def get_updated_model(log_file, data):
"""
:param name:
:param data:
:param labels:
:return:
>>> log_file = '/tmp/tf_examples/two_layer_final_model_DNN_32_128_10000_0.01/'
>>> data = data_dict_p
"""
layers, steps, lr = [32, 128], 10000, .01
model = skflow.TensorFlowDNNClassifier(hidden_units=layers,
n_classes=2,
batch_size=128,
steps=steps,
learning_rate=lr)
m = monitors.ValidationMonitor(data['X_train'],
data['y_train'].values,
every_n_steps=200)
model.fit(data['X'], list(data['y'].values), logdir=log_file)
_pred = model.predict(data['X'])
print accuracy_score(_pred, data['y'].values)
print confusion_matrix(_pred, data['y'])
return model
def train_inital_tf_model(data):
""" Run this only once, otherwise it take forever """
layers, steps, lr = [32, 128], 10000, .01
layers_str = str(layers).replace('[', '').replace(']',
'').replace(', ', '_')
log_dir = '/tmp/tf_examples/{}_DNN_{}_{}_{}/'.format(
"two_layer_final_model", layers_str, steps, lr)
model = skflow.TensorFlowDNNClassifier(hidden_units=layers,
n_classes=2,
batch_size=128,
steps=steps,
learning_rate=lr)
m = monitors.ValidationMonitor(data['X_train'],
data['y_train'].values,
every_n_steps=200)
model.fit(data['X_train'],
list(data['y_train'].values),
logdir=log_dir,
monitors=[m])
_pred = model.predict(data['X'])
print accuracy_score(_pred, data['y'].values)
print confusion_matrix(_pred, data['y'])
# model.save(log_dir)
return model, log_dir
def train_and_evaluate(self):
"""Interleaves training and evaluation.
The frequency of evaluation is controlled by the contructor arg
`min_eval_frequency`. When this parameter is None or 0, evaluation happens
only after training has completed. Note that evaluation cannot happen
more frequently than checkpoints are taken. If no new snapshots are
available when evaluation is supposed to occur, then evaluation doesn't
happen for another `min_eval_frequency` steps (assuming a checkpoint is
available at that point). Thus, settings `min_eval_frequency` to 1 means
that the model will be evaluated everytime there is a new checkpoint.
This is particular useful for a "Master" task in the cloud, whose
responsibility it is to take checkpoints, evaluate those checkpoints,
and write out summaries. Participating in training as the supervisor
allows such a task to accomplish the first and last items, while
performing evaluation allows for the second.
Returns:
The result of the `evaluate` call to the `Estimator` as well as the
export results using the specified `ExportStrategy`.
"""
# The directory to which evaluation summaries are written are determined
# by adding a suffix to 'eval'; that suffix is the 'name' parameter to
# the various evaluate(...) methods. By setting it to None, we force
# the directory name to simply be 'eval'.
eval_dir_suffix = None
# We set every_n_steps to 1, but evaluation only occurs when a new
# snapshot is available. If, by the time we finish evaluation
# there is a new snapshot, then we just evaluate again. Otherwise,
# we keep training until one becomes available.
with _new_attr_context(self, "_train_monitors"):
self._train_monitors = self._train_monitors or []
if self._min_eval_frequency:
self._train_monitors += [
monitors.ValidationMonitor(
input_fn=self._eval_input_fn,
eval_steps=self._eval_steps,
metrics=self._eval_metrics,
every_n_steps=self._min_eval_frequency,
name=eval_dir_suffix,
hooks=self._eval_hooks)
]
self.train(delay_secs=0)
eval_result = self._call_evaluate(input_fn=self._eval_input_fn,
steps=self._eval_steps,
metrics=self._eval_metrics,
name=eval_dir_suffix,
hooks=self._eval_hooks)
export_results = self._maybe_export(eval_result)
return eval_result, export_results
def local_run(self):
"""Run when called on local machine.
Returns:
The result of the `evaluate` call to the `Estimator`.
"""
self._train_monitors = self._train_monitors or []
if self._local_eval_frequency:
self._train_monitors += [monitors.ValidationMonitor(
input_fn=self._eval_input_fn, eval_steps=self._eval_steps,
metrics=self._eval_metrics, every_n_steps=self._local_eval_frequency
)]
self.train()
return self.evaluate()
def train_and_evaluate(self):
"""Interleaves training and evaluation.
The frequency of evaluation is controlled by the constructor arg
`min_eval_frequency`. When this parameter is 0, evaluation happens
only after training has completed. Note that evaluation cannot happen
more frequently than checkpoints are taken. If no new snapshots are
available when evaluation is supposed to occur, then evaluation doesn't
happen for another `min_eval_frequency` steps (assuming a checkpoint is
available at that point). Thus, settings `min_eval_frequency` to 1 means
that the model will be evaluated everytime there is a new checkpoint.
This is particular useful for a "Master" task in the cloud, whose
responsibility it is to take checkpoints, evaluate those checkpoints,
and write out summaries. Participating in training as the supervisor
allows such a task to accomplish the first and last items, while
performing evaluation allows for the second.
Returns:
The result of the `evaluate` call to the `Estimator` as well as the
export results using the specified `ExportStrategy`.
"""
# The directory to which evaluation summaries are written are determined
# by adding a suffix to 'eval'; that suffix is the 'name' parameter to
# the various evaluate(...) methods. By setting it to None, we force
# the directory name to simply be 'eval'.
eval_dir_suffix = None
# We set every_n_steps to 1, but evaluation only occurs when a new
# snapshot is available. If, by the time we finish evaluation
# there is a new snapshot, then we just evaluate again. Otherwise,
# we keep training until one becomes available.
with _new_attr_context(self, "_train_monitors"):
self._train_monitors = self._train_monitors or []
config = self._estimator.config
intermediate_export = self._checkpoint_and_export and (
config.save_checkpoints_secs or config.save_checkpoints_steps)
if intermediate_export:
# Create a partially specified evaluate function with the desired
# arguments. This will be executed by the _EvalAndExportListener,
# which will specify the latest checkpoint path.
eval_fn = functools.partial(self._call_evaluate,
input_fn=self._eval_input_fn,
steps=self._eval_steps,
metrics=self._eval_metrics,
hooks=self._eval_hooks)
export_listener = _EvalAndExportListener(
eval_fn=eval_fn,
export_fn=self._maybe_export,
model_dir=self._estimator.model_dir)
saver_hook = basic_session_run_hooks.CheckpointSaverHook(
checkpoint_dir=self._estimator.model_dir,
save_secs=config.save_checkpoints_secs,
save_steps=config.save_checkpoints_steps,
listeners=[export_listener])
self._train_monitors += [saver_hook]
else:
if self._min_eval_frequency:
self._train_monitors += [
monitors.ValidationMonitor(
input_fn=self._eval_input_fn,
eval_steps=self._eval_steps,
metrics=self._eval_metrics,
every_n_steps=self._min_eval_frequency,
name=eval_dir_suffix,
hooks=self._eval_hooks)
]
self.train(delay_secs=0)
# If the checkpoint_and_export flag and appropriate estimator configuration
# parameters are set, then model evaluations and exports are done during the
# training process. In particular, this will always occur at the end of
# training, so we return the most recent results to avoid performing a
# duplicate evaluation and model export.
if intermediate_export:
return export_listener.eval_result, export_listener.export_results
else:
eval_result = self._call_evaluate(input_fn=self._eval_input_fn,
steps=self._eval_steps,
metrics=self._eval_metrics,
name=eval_dir_suffix,
hooks=self._eval_hooks)
export_results = self._maybe_export(eval_result)
return eval_result, export_results
