def ExportMetaGraph(meta_graph):
"""Export the meta graph into a file under specific folder.
You can set the exporting prefix by `config.ExportMetaGraph(prefix)`_.
Parameters
----------
meta_graph : dragon_pb2.GraphDef
The definition of meta graph.
Returns
-------
None
"""
from dragon.config import option, logger
if option['export_meta_graph']:
if not os.path.exists(option['export_meta_graph']):
try:
os.makedirs(option['export_meta_graph'])
except Exception:
raise ValueError('The given prefix is invalid.')
filepath = os.path.join(option['export_meta_graph'],
meta_graph.name + '.metatxt')
with open(filepath, 'w') as f:
f.write(str(meta_graph))
logger.info('Export meta graph into: {}'.format(filepath))
def load_state_dict(self, state_dict, strict=True, verbose=True):
if verbose: logger.info('Load the state dict.')
def submodule_key_mismatch(full_name, is_missing):
module = self
names = full_name.split(".")
for module_name in names[:-1]:
if module_name in module._modules:
module = module._modules[module_name]
else:
return
module._load_state_dict_key_mismatch(full_name, names[-1],
is_missing)
unexpected = []
own_state = self.state_dict()
for name, param in state_dict.items():
if name in own_state:
state_shape = own_state[name].shape
param_shape = param.shape
if state_shape != param_shape:
raise ValueError(
'Size of state({}) is ({}), \n'
'While load from Size of ({}).'.format(
name, ', '.join([str(d) for d in state_shape]),
', '.join([str(d) for d in param_shape])))
if own_state[name].dtype != str(param.dtype):
raise ValueError('DType of state({}) is {}, \n'
'While load from a PyArray of {}.'.format(
name, own_state[name].dtype,
str(param.dtype)))
if isinstance(param, Tensor):
own_state[name].copy_(param)
elif isinstance(param, np.ndarray):
dg.tensor_utils.SetPyArray(own_state[name], param)
else:
raise ValueError(
'Excepted the type of source state is either '
'torch.Tensor or numpy.ndarray, got {}.'.format(
type(param)))
if verbose:
logger.info('* Tensor({}) loaded, Size: ({})'.format(
name, ', '.join([str(d) for d in param_shape])))
if strict:
missing = set(own_state.keys()) - set(state_dict.keys())
# pass the mismatch info to submodules so that they have a chance to
# raise a custom class-specific error
for name in unexpected:
submodule_key_mismatch(name, False)
for name in missing:
submodule_key_mismatch(name, True)
error_msg = ''
if len(unexpected) > 0:
error_msg += 'Unexpected key(s) in state_dict: {}. '.format(
', '.join('"{}"'.format(k) for k in unexpected))
if len(missing) > 0:
error_msg += 'Missing key(s) in state_dict: {}. '.format(
', '.join('"{}"'.format(k) for k in missing))
if len(error_msg) > 0:
raise KeyError(error_msg)
def GetOptimizedGraph(meta_graph):
"""Return the optimized graph.
Parameters
----------
meta_graph : dragon_pb2.GraphDef
The definition of meta graph.
Returns
-------
graph_def : dragon_pb2.GraphDef
The definition of optimized graph.
"""
from dragon.config import logger
graph_name = meta_graph.name
graph_tensor = 'GraphDef_' + graph_name
if not HasTensorCC(graph_tensor):
logger.info(
'Graph({}) does not exist, ignore printing....'.format(graph_name))
return
opt_graph_def = pb.GraphDef()
opt_graph_def.ParseFromString(FetchTensor(graph_tensor))
return opt_graph_def
def GetLearningRate(self):
"""Get learning rate based on the preset policy.
Returns
-------
None
References
----------
The implementation of `GetLearningRate(solver.cpp, L27)`_.
"""
from dragon.config import logger
policy = self._param.lr_policy
if policy == "step":
new_step = int(self._iter / self._param.stepsize)
if self._current_step != new_step:
new_lr = self._param.base_lr * pow(self._param.gamma, new_step)
self._current_step = new_step
self._optimizer.lr = new_lr
if policy == 'multistep':
if self._current_step < len(self._param.stepvalue) \
and self._iter >= self._param.stepvalue[self._current_step]:
self._current_step = self._current_step + 1
logger.info('MultiStep Status: Iteration {}, step = {}' \
.format(self._iter, self._current_step))
new_lr = self._param.base_lr * \
pow(self._param.gamma, self._current_step)
self._optimizer.lr = new_lr
if policy == 'multifixed':
stage_lrs = self._param.stage_lr
stage_iters = self._param.stage_iter
if self._iter < stage_iters[self._current_step]:
self._optimizer.lr = stage_lrs[self._current_step]
else:
if self._current_step + 1 < len(stage_iters):
self._current_step = self._current_step + 1
logger.info('MultiFixed Status: Iteration {}, stage = {}' \
.format(self._iter, self._current_step))
self._optimizer.lr = stage_lrs[self._current_step]
if policy == 'inv':
power = self._param.power
gamma = self._param.gamma
self._optimizer.lr = self._param.base_lr * \
pow(1.0 + gamma * self._iter, -power)
if policy == 'poly':
power = self._param.power
max_iter = self._param.max_iter
self._optimizer.lr = self._param.base_lr * \
pow(1.0 - float(self.iter) / max_iter, power)
def Snapshot(
tensors, filename,
prefix='', suffix='.bin',
format='default'):
"""Snapshot tensors into a binary file.
Parameters
----------
tensors : list of Tensor or Tensor
The tensors to be wrote.
filename : str
The name of this binary file.
prefix : str
The prefix of this binary file.
suffix : str
The suffix of this binary file.
format : str
The format of this binary file.
Returns
-------
None
Notes
-----
The full file path will be: ``prefix`` + ``filename`` + ``suffix``.
Available formats: ['default', 'caffe'].
"""
from dragon.config import logger
file_path = prefix + filename + suffix
if mpi.Is_Init():
if not mpi.AllowSnapshot(): return
file_path = file_path + '.rank.{}'.format(mpi.Rank())
dir = os.path.split(file_path)[0]
if len(dir) > 0 and not os.path.exists(dir): os.makedirs(dir)
if format == 'default':
state_dict = {}
for tensor in tensors:
state_dict[tensor.name] = FetchTensor(tensor)
with open(file_path, 'wb') as f:
cPickle.dump(state_dict, f, cPickle.HIGHEST_PROTOCOL)
logger.info('Snapshot Model@: ' + file_path)
logger.info('Model Format: cPickle')
elif format is 'caffe':
names = [tensor.name for tensor in tensors]
SnapshotCC(file_path, names, 1)
else: raise TypeError('Unknown binary format: {}'.format(format))
def snapshot(self):
if mpi.Is_Init():
if not mpi.AllowSnapshot(): return
net = self.solver.net
infix = ('_' + cfg.TRAIN.SNAPSHOT_INFIX
if cfg.TRAIN.SNAPSHOT_INFIX != '' else '')
filename = (self.solver_param.snapshot_prefix + infix +
'_iter_{:d}'.format(self.solver.iter) + '.caffemodel')
filename = os.path.join(self.output_dir, filename)
net.save(str(filename))
logger.info('Wrote snapshot to: {:s}'.format(filename))
return filename
def PrintOptimizedGraph(graph_def):
graph_name = graph_def.name
graph_tensor = 'GraphDef_' + graph_name
if not HasTensorCC(graph_tensor):
logger.info(
'graph: {} does not exist, ignore printing....'.format(graph_name))
return
graph_def = pb.GraphDef()
graph_def.ParseFromString(FetchTensor(graph_tensor))
logger.info(graph_def)
def step(self, iters):
""" simply follow the pycaffe style """
start_iter = self._iter; stop_iter = self._iter + iters
loss_vec = []; smoothed_loss = 0
tic = time.time()
while self._iter < stop_iter:
# test if necessary
if self._param.test_interval and \
self._iter % self._param.test_interval == 0:
if (self._iter == 0 and
self._param.test_initialization) or self._iter != 0:
for test_id in xrange(len(self.tests)): self.Test(test_id)
# forward & backward & compute_loss
loss = 0.0
for i in xrange(self._param.iter_size):
self.train(return_outputs=False)
if root_solver():
for cost in self._net._costs: loss += FetchTensor(cost)[0]
if root_solver():
loss /= self._param.iter_size
if len(loss_vec) < self._param.average_loss:
loss_vec.append(loss)
smoothed_loss = (smoothed_loss * (len(loss_vec) - 1) + loss) / len(loss_vec);
else:
idx = (self._iter - start_iter) % self._param.average_loss
smoothed_loss += ((loss - loss_vec[idx]) / self._param.average_loss)
loss_vec[idx] = loss
# apply update
self.CheckLearningRate()
self.update()
# display
if root_solver() and self._param.display:
if self._iter % self._param.display == 0:
base_lr = self._updater.lr
logger.info('Iteration %d, lr = %s, loss = %f, time = %.2fs' % \
(self._iter, str(base_lr), smoothed_loss, time.time() - tic))
tic = time.time()
for idx, net_output in enumerate(self._net._net_outputs):
vals = FetchTensor(self._net.blobs[net_output].data)
for val in vals:
logger.info(' Train net output #{}({}): {}'.format(idx, net_output, val))
self.scalar_writer.add_summary((net_output, val), self._iter)
self._iter = self._iter + 1
# snapshot
if self._param.snapshot:
if self._iter % self._param.snapshot == 0: self.snapshot()
def Restore(filename, format=0):
if mpi.is_init():
if not mpi.allow_snapshot():
if not mpi.allow_parallel():
filename += '.rank.{}'.format(mpi.rank())
return
assert os.path.exists(
filename), 'model of path({}) does not exist.'.format(filename)
if format is 0:
content = cPickle.load(open(filename, 'rb'))
logger.info('Restore From Model@: ' + filename)
logger.info('Model Format: cPickle')
for key, ndarray in content.items():
if not HasTensor(key):
logger.info(
'[Warning]: Tensor({}) of model does not exist in any Graphs, skip.'
.format(key))
else:
logger.info('[Info]: Tensor({}) restored.'.format(key))
FeedTensor(key, ndarray)
elif format is 1:
# TODO(PhyscalX): caffemodel can't save the tensor name
# TODO(PhyscalX): we simply use 'Scope + LayerName + @paramX'
RestoreCC(filename, '', format)
def LogMetaGraph(meta_graph):
"""Log the meta graph.
Parameters
----------
meta_graph : dragon_pb2.GraphDef
The definition of meta graph.
Returns
-------
None
"""
from dragon.config import option, logger
if option['log_meta_graph']: logger.info(meta_graph)
def Test(self, test_idx):
"""Test the specific net.
Parameters
----------
test_idx : int
The idx of test net.
Returns
-------
None
References
----------
The implementation of `Test(solver.cpp, L328)`_.
"""
from dragon.config import logger
test_score = []
output_id = []
test_iter = self._param.test_iter[test_idx]
net = self._test_nets[test_idx]
for iter in xrange(test_iter):
self.tests[test_idx](return_outputs=False)
if not root_solver(): continue
if iter == 0:
for net_output in net._net_outputs:
vals = ws.FetchTensor(net.blobs[net_output].data)
for idx, val in enumerate(vals):
test_score.append(val)
output_id.append(net_output)
else:
i = 0
for net_output in net._net_outputs:
vals = ws.FetchTensor(net.blobs[net_output].data)
for idx, val in enumerate(vals):
test_score[i] += val
i += 1
if not root_solver(): return
logger.info('Iteration {}, Test net #{}'.format(self._iter, test_idx))
for idx, score in enumerate(test_score):
logger.info(' Test net output #%d(%s): %.4f' %
(idx, output_id[idx], score / test_iter))
self.scalar_writer.add_summary((output_id[idx], score / test_iter),
self._iter)
def LogOptimizedGraph(meta_graph):
"""Log the optimized graph.
Parameters
----------
meta_graph : dragon_pb2.GraphDef
The definition of meta graph.
Returns
-------
None
"""
from dragon.config import option, logger
if option['log_optimized_graph']:
optimized_graph = GetOptimizedGraph(meta_graph)
logger.info(optimized_graph)
def Test(self, test_idx):
"""Test the specific net.
Parameters
----------
test_idx : int
The idx of test net.
Returns
-------
None
References
----------
The implementation of `Test(solver.cpp, L328)`_.
"""
from dragon.config import logger
test_score = []
output_id = []
test_iter = self._param.test_iter[test_idx]
net = self._test_nets[test_idx]
for iter in xrange(test_iter):
self.tests[test_idx](return_outputs=False)
if not root_solver(): continue
if iter == 0:
for net_output in net._net_outputs:
vals = ws.FetchTensor(net.blobs[net_output].data)
for idx, val in enumerate(vals):
test_score.append(val)
output_id.append(net_output)
else:
i = 0
for net_output in net._net_outputs:
vals = ws.FetchTensor(net.blobs[net_output].data)
for idx, val in enumerate(vals):
test_score[i] += val
i += 1
if not root_solver(): return
logger.info('Iteration {}, Test net #{}'.format(self._iter, test_idx))
for idx, score in enumerate(test_score):
logger.info(' Test net output #%d(%s): %.4f' % (idx, output_id[idx], score / test_iter))
self.scalar_writer.add_summary((output_id[idx], score / test_iter), self._iter)
def CheckLearningRate(self):
policy = self._param.lr_policy
if policy == "step":
new_step = int(self._iter / self._param.stepsize)
if self._current_step != new_step:
new_lr = self._param.base_lr * pow(self._param.gamma, new_step)
self._current_step = new_step
self._updater.lr = new_lr
if policy == 'multistep':
if self._current_step < len(self._param.stepvalue) \
and self._iter >= self._param.stepvalue[self._current_step]:
self._current_step = self._current_step + 1
logger.info('MultiStep Status: Iteration {}, step = {}' \
.format(self._iter, self._current_step))
new_lr = self._param.base_lr * \
pow(self._param.gamma, self._current_step)
self._updater.lr = new_lr
if policy == 'multifixed':
stage_lrs = self._param.stage_lr
stage_iters = self._param.stage_iter
if self._iter < stage_iters[self._current_step]:
self._updater.lr = stage_lrs[self._current_step]
else:
if self._current_step + 1 < len(stage_iters):
self._current_step = self._current_step + 1
logger.info('MultiFixed Status: Iteration {}, stage = {}' \
.format(self._iter, self._current_step))
self._updater.lr = stage_lrs[self._current_step]
if policy == 'inv':
power = self._param.power
gamma = self._param.gamma
self._updater.lr = self._param.base_lr * \
pow(1.0 + gamma * self._iter, -power)
if policy == 'poly':
power = self._param.power
max_iter = self._param.max_iter
self._updater.lr = self._param.base_lr * \
pow(1.0 - float(self.iter) / max_iter, power)
def train_net(solver_txt,
output_dir,
pretrained_model=None,
snapshot_model=None,
start_iter=0,
max_iters=60000,
warm_up=0):
sw = SolverWrapper(solver_txt,
output_dir,
pretrained_model=pretrained_model)
if snapshot_model is not None:
sw.restore(start_iter, snapshot_model)
logger.info('Solving...')
model_paths = sw.train_model(max_iters, warm_up)
logger.info('done solving')
return model_paths
def export(self, name=None, export_dir='./'):
"""Export the meta graph of this defined function.
Parameters
----------
export_dir : str
The directory to export the meta text file.
"""
from dragon.config import logger
if not os.path.exists(export_dir):
try:
os.makedirs(export_dir)
except Exception:
raise ValueError('The given directory can not be created.')
meta_graph_copy = copy.deepcopy(self.meta_graph)
meta_graph_copy.name = self.meta_graph.name if name is None else name
file = os.path.join(export_dir, meta_graph_copy.name + '.metatxt')
with open(file, 'w') as f:
f.write(str(meta_graph_copy))
logger.info('Export meta graph into: {}'.format(file))
def echo(self):
logger.info(
'---------------------------------------------------------')
logger.info('Optimizer: {}, Using config:'.format(
self._type.split('Update')[0]))
pprint.pprint(self._hyper_params)
logger.info(
'---------------------------------------------------------')
def train_model(self, max_iters, warm_up):
last_snapshot_iter = -1
timer = Timer()
model_paths = []
# lower the learning rate then warm-up
if warm_up != 0:
self.solver._optimizer.lr *= self.solver_param.gamma
while self.solver.iter < max_iters:
if warm_up > 0 and self.solver.iter == warm_up:
self.solver._optimizer.lr /= self.solver_param.gamma
timer.tic()
self.solver.step(1)
timer.toc()
if self.solver.iter % (10 * self.solver_param.display) == 0:
logger.info('speed: {:.3f}s / iter'.format(timer.average_time))
if self.solver.iter % cfg.TRAIN.SNAPSHOT_ITERS == 0:
last_snapshot_iter = self.solver.iter
model_paths.append(self.snapshot())
if last_snapshot_iter != self.solver.iter:
model_paths.append(self.snapshot())
return model_paths
def Snapshot(tensors, filename, prefix='', suffix='.bin', format=0):
filepath = prefix + filename + suffix
if mpi.is_init():
if not mpi.allow_snapshot(): return
filepath += '.rank.{}'.format(mpi.rank())
dir = os.path.split(filepath)[0]
if len(dir) > 0 and not os.path.exists(dir): os.makedirs(dir)
if format is 0:
# kv-store
content = {}
for tensor in tensors:
content[tensor.name] = FetchTensor(tensor)
with open(filepath, 'wb') as f:
cPickle.dump(content, f, cPickle.HIGHEST_PROTOCOL)
logger.info('Snapshot Model@: ' + filepath)
logger.info('Model Format: cPickle')
elif format is 1:
# caffe-store
names = [tensor.name for tensor in tensors]
SnapshotCC(filepath, names, format)
def echo(self):
"""
Print Updater Information.
"""
from dragon.config import logger
logger.info('---------------------------------------------------------')
logger.info('Optimizer: {}, Using config:'.format(self._type.split('Update')[0]))
pprint.pprint(self._hyper_params)
logger.info('---------------------------------------------------------')
def echo(self):
"""
Print Updater Information.
"""
from dragon.config import logger
logger.info('---------------------------------------------------------')
logger.info('Optimizer: {}, Using config:'.format(self._type.split('Update')[0]))
pprint.pprint(self._hyper_params)
logger.info('---------------------------------------------------------')
def Restore(binary_file, format='default'):
"""Restore tensors from a binary file.
Parameters
----------
binary_file : str
The path of binary file.
format : str
The format of this binary file.
Returns
-------
None
Notes
-----
Available formats: ['default', 'caffe'].
"""
from dragon.config import logger
assert os.path.exists(binary_file), \
'Binary file({}) does not exist.'.format(binary_file)
if format == 'default':
try:
state_dict = cPickle.load(open(binary_file, 'rb'))
except UnicodeDecodeError:
state_dict = cPickle.load(open(binary_file, 'rb'),
encoding='iso-8859-1')
logger.info('Restore From Model@: ' + binary_file)
logger.info('Model Format: cPickle')
for k, v in state_dict.items():
if not HasTensor(k):
logger.info(
'[Warning]: Tensor({}) does not exist in any Graphs, skip.'
.format(k))
else:
FeedTensor(k, v)
logger.info('[Info]: Tensor({}) is restored.'.format(k))
elif format == 'caffe':
# TODO(PhyscalX): caffe models can't save the tensor name
# TODO(PhyscalX): we simply use layer_name + @paramX
RestoreCC(binary_file, 1)
else:
raise TypeError('Unknown binary format: {}'.format(format))
def Test(self, test_idx):
test_score = []; output_id = []
test_iter = self._param.test_iter[test_idx]
net = self._test_nets[test_idx]
for iter in xrange(test_iter):
self.tests[test_idx](return_outputs=False)
if not root_solver(): continue
if iter == 0:
for net_output in net._net_outputs:
vals = FetchTensor(net.blobs[net_output].data)
for idx, val in enumerate(vals):
test_score.append(val)
output_id.append(net_output)
else:
i = 0
for net_output in net._net_outputs:
vals = FetchTensor(net.blobs[net_output].data)
for idx, val in enumerate(vals):
test_score[i] += val; i = i + 1
if not root_solver(): return
logger.info('Iteration {}, Test net #{}'.format(self._iter, test_idx))
for idx, score in enumerate(test_score):
logger.info(' Test net output #%d(%s): %.4f' % (idx, output_id[idx], score / test_iter))
self.scalar_writer.add_summary((output_id[idx], score / test_iter), self._iter)
def Restore(filepath, format='default'):
"""Restore tensors from a binary file.
Parameters
----------
filepath : str
The path of binary file.
format : str
The format of this binary file.
Returns
-------
None
Notes
-----
Available formats: ['default', 'caffe'].
"""
from dragon.config import logger
assert os.path.exists(
filepath), 'model of path({}) does not exist.'.format(filepath)
if format == 'default':
try:
content = cPickle.load(open(filepath, 'rb'))
except UnicodeDecodeError:
content = cPickle.load(open(filepath, 'rb'), encoding='iso-8859-1')
logger.info('Restore From Model@: ' + filepath)
logger.info('Model Format: cPickle')
for key, ndarray in content.items():
if not HasTensor(key):
logger.info(
'[Warning]: Tensor({}) of model does not exist in any Graphs, skip.'
.format(key))
else:
logger.info('[Info]: Tensor({}) restored.'.format(key))
FeedTensor(key, ndarray)
elif format == 'caffe':
# TODO(PhyscalX): caffemodel can't save the tensor name
# TODO(PhyscalX): we simply use layer_name + @paramX
RestoreCC(filepath, 1)
else:
raise TypeError('Unknown binary format: {}'.format(format))
def cleanup():
def terminate(processes):
for process in processes:
process.terminate()
process.join()
from dragon.config import logger
logger.info('Terminating BlobFetcher ......')
terminate(self._fetchers)
logger.info('Terminating DataTransformer ......')
terminate(self._transformers)
logger.info('Terminating DataReader......')
terminate(self._readers)
def echo(self):
logger.info(
'---------------------------------------------------------')
logger.info('BatchReader, Using config:')
params = {
'prefetching': self._prefetch,
'num_readers': self._num_readers,
'num_transformers': self._num_transformers,
'num_fetchers': self._num_fetchers
}
pprint.pprint(params)
logger.info(
'---------------------------------------------------------')
def echo(self):
"""
Print I/O Information.
"""
from dragon.config import logger
logger.info('---------------------------------------------------------')
logger.info('BatchReader, Using config:')
params = {'prefetching': self._prefetch,
'num_readers': self._num_readers,
'num_transformers': self._num_transformers,
'num_fetchers': self._num_fetchers}
pprint.pprint(params)
logger.info('---------------------------------------------------------')
def cleanup():
def terminate(processes):
for process in processes:
process.terminate()
process.join()
from dragon.config import logger
logger.info('Terminating BlobFetcher ......')
terminate(self._fetchers)
logger.info('Terminating DataTransformer ......')
terminate(self._transformers)
logger.info('Terminating DataReader......')
terminate(self._readers)
def echo(self):
"""
Print I/O Information.
"""
from dragon.config import logger
logger.info(
'---------------------------------------------------------')
logger.info('BatchReader, Using config:')
params = {
'prefetching': self._prefetch,
'num_readers': self._num_readers,
'num_transformers': self._num_transformers,
'num_fetchers': self._num_fetchers
}
pprint.pprint(params)
logger.info(
'---------------------------------------------------------')
def register_in_workspace(self):
if not self._registered:
for k, v in self._defaults.items():
# convert all defaults as float32 for convenience
ws.FeedTensor(self._slot + "/" + k,
np.array([v], dtype=np.float32))
self._registered = True
if self._verbose:
from dragon.config import logger
logger.info(
'---------------------------------------------------------'
)
logger.info('Optimizer: {}, Using config:'.format(
self.type(True)))
pprint.pprint(self._defaults)
logger.info(
'---------------------------------------------------------'
)
def register_in_workspace(self):
if not self._registered:
for k, v in self._defaults.items():
ws.FeedTensor(self._slot + "/" + k,
v,
dtype='float32',
force_cpu=True)
self._registered = True
if self._verbose:
from dragon.config import logger
logger.info(
'---------------------------------------------------------'
)
logger.info('Optimizer: {}, Using config:'.format(
self.type(True)))
pprint.pprint(self._defaults)
logger.info(
'---------------------------------------------------------'
)
def RunGradientFlow(input_flow, targets, input_grads=None, ignored_grads=None):
"""Compute the gradients of given input flows.
Parameters
----------
input_flow : list of OperatorDef or GraphDef
The referring flows to generate gradient flows.
targets : list or str
The solving targets, generate grads automatically.
input_grads : None or list of str
The input grads.
ignored_grads : None or list of str
The grads that are explicitly ignored.
Returns
-------
None
"""
if isinstance(input_flow, list):
graph_wrapper = pb.GraphDef()
graph_wrapper.op.extend(input_flow)
input_flow = graph_wrapper
if not isinstance(input_flow, pb.GraphDef):
raise TypeError('Excepted the type of input flow is either'
'a list of OperatorDef or a GraphDef, got {}.'.format(
type(input_flow)))
from dragon.config import option, logger
log_flow = True if option['log_optimized_graph'] or option[
'log_meta_graph'] else False
RunGradientFlowCC(_stringify_proto(input_flow), targets,
input_grads if input_grads else [],
ignored_grads if ignored_grads else [],
option['share_grads'], log_flow)
if log_flow:
g_flow = pb.GraphDef()
g_flow.ParseFromString(
FetchTensor('/export/dynamic_graph/gradient_flow'))
logger.info('>>>>>>>>>>>>>>>>>> Gradient Flow <<<<<<<<<<<<<<<<<<\n')
logger.info(g_flow)
logger.info('>>>>>>>>>>>>>>>>>> Gradient Flow <<<<<<<<<<<<<<<<<<\n')
def cleanup():
logger.info('Terminating Fetcher......')
self.terminate()
self.join()
def run(self, inputs, outputs):
"""
Run implement(i.e. forward-pass).
Parameters
----------
inputs : sequence of strs
Indicating the operator's inputs
outputs : sequence of strs
Indicating the operator's outputs
Returns
-------
None
"""
ws.FeedTensor(outputs[0], self._queue.get())
if __name__ == '__main__':
# def
y = ops.Run([], module=__name__, op='DataProcess', nout=1)
foo = theano.function(outputs=y)
# run
foo()
# fetch
logger.info('y \n-------------- \n', y.get_value(), '\n')
def step(self, iters):
"""Step the train net. [**PyCaffe Style**]
Parameters
----------
iters : int
The number of iterations to step.
Returns
-------
None
References
----------
The implementation of `Step(solver.cpp, L180)`_.
"""
from dragon.config import logger
start_iter = self._iter; stop_iter = self._iter + iters
loss_vec = []; smoothed_loss = 0
tic = time.time()
while self._iter < stop_iter:
# test if necessary
if self._param.test_interval and \
self._iter % self._param.test_interval == 0:
if (self._iter == 0 and
self._param.test_initialization) or self._iter != 0:
for test_id in xrange(len(self.tests)): self.Test(test_id)
# forward & backward & compute_loss
loss = 0.0
for i in xrange(self._param.iter_size):
self.train(return_outputs=False)
if root_solver():
for cost in self._net._costs:
cost_value = ws.FetchTensor(cost)
if cost_value.size == 1:
loss += cost_value[0]
if root_solver():
loss /= self._param.iter_size
if len(loss_vec) < self._param.average_loss:
loss_vec.append(loss)
smoothed_loss = (smoothed_loss * (len(loss_vec) - 1) + loss) / len(loss_vec);
else:
idx = (self._iter - start_iter) % self._param.average_loss
smoothed_loss += ((loss - loss_vec[idx]) / self._param.average_loss)
loss_vec[idx] = loss
# apply update
self.GetLearningRate()
self.update()
# display
if root_solver() and self._param.display:
if self._iter % self._param.display == 0:
base_lr = self._optimizer.lr
logger.info('Iteration %d, lr = %s, loss = %f, time = %.2fs' % \
(self._iter, str(base_lr), smoothed_loss, time.time() - tic))
tic = time.time()
for idx, net_output in enumerate(self._net.outputs):
vals = ws.FetchTensor(self._net.blobs[net_output].data)
for val in vals:
logger.info(' Train net output #{}({}): {}'.format(idx, net_output, val))
self.scalar_writer.add_summary((net_output, val), self._iter)
self._iter = self._iter + 1
# snapshot
if self._param.snapshot:
if self._iter % self._param.snapshot == 0: self.snapshot()
def cleanup():
from dragon.config import logger
logger.info('Terminating DragonBoard......')
self.terminate()
self.join()