def set_learning_rate(self, lr, exp_decay=None):
"""
:param lr:
:param exp_decay: a dictionary like dict(decay_steps=None, decay_rate=None, staircase=False, name=None),
otherwise None.
:return:
"""
if isinstance(exp_decay, dict):
if 'decay_steps' not in exp_decay:
raise KeyError(
'Set decay_steps in exp_decay=dict(decay_steps)')
if 'decay_rate' not in exp_decay:
raise KeyError('Set decay_steps in exp_decay=dict(decay_rate)')
if 'staircase' not in exp_decay:
exp_decay['staircase'] = False
if 'name' not in exp_decay:
exp_decay['name'] = None
self._lr = tfnn.train.exponential_decay(
lr,
self.global_step,
decay_steps=exp_decay['decay_steps'],
decay_rate=exp_decay['decay_rate'],
staircase=exp_decay['staircase'],
name=exp_decay['name'])
else:
self._lr = tfnn.constant(lr)
tfnn.scalar_summary('learning_rate', self._lr)
def _init_loss(self):
with tfnn.name_scope('predictions'):
if self.method == 'softmax':
self.predictions = tfnn.nn.softmax(self.layers_results['final'][-1], name='predictions')
elif self.method == 'sigmoid':
self.predictions = tfnn.nn.sigmoid(self.layers_results['final'][-1], name='predictions')
with tfnn.name_scope('loss'):
if self.method == 'softmax':
self.cross_entropy = tfnn.nn.softmax_cross_entropy_with_logits(
self.layers_results['final'][-1],
self.target_placeholder,
name='xentropy')
elif self.method == 'sigmoid':
self.cross_entropy = tfnn.nn.sigmoid_cross_entropy_with_logits(
self.layers_results['final'][-1],
self.target_placeholder,
name='xentropy')
else:
raise ValueError("method should be one of ['sparse_softmax', 'softmax', 'sigmoid']")
self.loss = tfnn.reduce_mean(self.cross_entropy, name='xentropy_mean')
if self.reg == 'l2':
with tfnn.name_scope('l2_reg'):
regularizers = 0
for layer in self.layers_results['Layer'][1:]:
regularizers += tfnn.nn.l2_loss(layer.W, name='l2_reg')
regularizers *= self.l2_placeholder
with tfnn.name_scope('l2_loss'):
self.loss += regularizers
tfnn.scalar_summary('loss', self.loss)
def __init__(self, network, ):
self.network = network
if isinstance(self.network, tfnn.ClfNetwork):
with tfnn.name_scope('accuracy'):
with tfnn.name_scope('correct_prediction'):
correct_prediction = tfnn.equal(tfnn.argmax(network.predictions, 1),
tfnn.argmax(network.target_placeholder, 1), name='correct_prediction')
with tfnn.name_scope('accuracy'):
self.accuracy = tfnn.reduce_mean(tfnn.cast(correct_prediction, tfnn.float32), name='accuracy')
tfnn.scalar_summary('accuracy', self.accuracy)
elif isinstance(self.network, tfnn.RegNetwork):
self.first_time_lm = True
self.first_time_soc = True
with tfnn.name_scope('r2_score'):
with tfnn.name_scope('ys_mean'):
ys_mean = tfnn.reduce_mean(network.target_placeholder, reduction_indices=[0], name='ys_mean')
with tfnn.name_scope('total_sum_squares'):
ss_tot = tfnn.reduce_sum(tfnn.square(network.target_placeholder - ys_mean),
reduction_indices=[0], name='total_sum_squares')
# ss_reg = np.sum(np.square(predictions-ys_mean), axis=0)
with tfnn.name_scope('residual_sum_squares'):
ss_res = tfnn.reduce_sum(tfnn.square(network.target_placeholder - network.predictions),
reduction_indices=[0], name='residual_sum_squares')
with tfnn.name_scope('coefficient_of_determination'):
self.r2_score = tfnn.sub(tfnn.constant(1, dtype=tfnn.float32), (ss_res / ss_tot)[0],
name='coefficient_of_determination')
tfnn.scalar_summary('r2_score', self.r2_score)
def set_learning_rate(self, lr, exp_decay=None):
"""
:param lr:
:param exp_decay: a dictionary like dict(decay_steps=None, decay_rate=None, staircase=False, name=None),
otherwise None.
:return:
"""
if isinstance(exp_decay, dict):
if 'decay_steps' not in exp_decay:
raise KeyError('Set decay_steps in exp_decay=dict(decay_steps)')
if 'decay_rate' not in exp_decay:
raise KeyError('Set decay_steps in exp_decay=dict(decay_rate)')
if 'staircase' not in exp_decay:
exp_decay['staircase'] = False
if 'name' not in exp_decay:
exp_decay['name'] = None
self._lr = tfnn.train.exponential_decay(lr, self.global_step,
decay_steps=exp_decay['decay_steps'],
decay_rate=exp_decay['decay_rate'],
staircase=exp_decay['staircase'],
name=exp_decay['name'])
else:
self._lr = tfnn.constant(lr)
tfnn.scalar_summary('learning_rate', self._lr)
def _set_accuracy(self):
if isinstance(self.network, tfnn.ClfNetwork):
with tfnn.name_scope('accuracy'):
correct_prediction = tfnn.equal(
tfnn.argmax(self.network.predictions, 1),
tfnn.argmax(self.network.target_placeholder, 1),
name='correct_prediction')
self.accuracy = tfnn.reduce_mean(
tfnn.cast(correct_prediction, tfnn.float32), name='accuracy')
tfnn.scalar_summary('accuracy', self.accuracy)
def __init__(self, input_size, output_size, do_dropout, do_l2, ntype):
self.normalizer = Normalizer()
self.input_size = input_size
self.output_size = output_size
self.global_step = tfnn.Variable(0, trainable=False)
if do_dropout and do_l2:
raise ValueError(
'Cannot do dropout and l2 at once. Choose only one of them.')
if do_dropout:
self.reg = 'dropout'
if do_l2:
self.reg = 'l2'
if (do_dropout is False) & (do_l2 is False):
self.reg = None
with tfnn.name_scope('inputs'):
self.data_placeholder = tfnn.placeholder(
dtype=tfnn.float32,
shape=[None, self.input_size],
name='x_input')
self.target_placeholder = tfnn.placeholder(
dtype=tfnn.float32,
shape=[None, self.output_size],
name='y_input')
if do_dropout:
self.keep_prob_placeholder = tfnn.placeholder(
dtype=tfnn.float32)
tfnn.scalar_summary('dropout_keep_probability',
self.keep_prob_placeholder)
_reg_value = self.keep_prob_placeholder
elif do_l2:
self.l2_placeholder = tfnn.placeholder(tfnn.float32)
tfnn.scalar_summary('l2_value', self.l2_placeholder)
_reg_value = self.l2_placeholder
else:
_reg_value = None
self.layers_configs = {
'type': ['input'],
'name': ['input_layer'],
'neural_structure': [{
'input_size': self.input_size,
'output_size': self.input_size
}],
'ntype':
ntype,
}
self.layers_results = {
'reg_value': _reg_value,
'Layer': [None],
'Wx_plus_b': [None],
'activated': [None],
'dropped': [None],
'final': [self.data_placeholder]
}
def _set_accuracy(self):
if isinstance(self.network, tfnn.ClfNetwork):
with tfnn.name_scope('accuracy'):
correct_prediction = tfnn.equal(
tfnn.argmax(self.network.predictions, 1),
tfnn.argmax(self.network.target_placeholder, 1),
name='correct_prediction')
self.accuracy = tfnn.reduce_mean(tfnn.cast(
correct_prediction, tfnn.float32),
name='accuracy')
tfnn.scalar_summary('accuracy', self.accuracy)
def __init__(self, input_size, output_size, do_dropout, do_l2, ntype):
self.normalizer = Normalizer()
self.input_size = input_size
self.output_size = output_size
self.global_step = tfnn.Variable(0, trainable=False)
if do_dropout and do_l2:
raise ValueError('Cannot do dropout and l2 at once. Choose only one of them.')
if do_dropout:
self.reg = 'dropout'
if do_l2:
self.reg = 'l2'
if (do_dropout is False) & (do_l2 is False):
self.reg = None
with tfnn.name_scope('inputs'):
self.data_placeholder = tfnn.placeholder(dtype=tfnn.float32,
shape=[None, self.input_size],
name='x_input')
self.target_placeholder = tfnn.placeholder(dtype=tfnn.float32,
shape=[None, self.output_size],
name='y_input')
if do_dropout:
self.keep_prob_placeholder = tfnn.placeholder(dtype=tfnn.float32)
tfnn.scalar_summary('dropout_keep_probability', self.keep_prob_placeholder)
_reg_value = self.keep_prob_placeholder
elif do_l2:
self.l2_placeholder = tfnn.placeholder(tfnn.float32)
tfnn.scalar_summary('l2_value', self.l2_placeholder)
_reg_value = self.l2_placeholder
else:
_reg_value = None
self.layers_configs = {
'type': ['input'],
'name': ['input_layer'],
'neural_structure': [{'input_size': self.input_size, 'output_size': self.input_size}],
'ntype': ntype,
}
self.layers_results = {
'reg_value': _reg_value,
'Layer': [None],
'Wx_plus_b': [None],
'activated': [None],
'dropped': [None],
'final': [self.data_placeholder]
}
def _init_loss(self):
with tfnn.name_scope('predictions'):
self.predictions = self.layers_final_output.iloc[-1] + 0
with tfnn.name_scope('loss'):
loss_square = tfnn.square(self.target_placeholder - self.layers_final_output.iloc[-1], name='loss_square')
loss_sum = tfnn.reduce_sum(loss_square, reduction_indices=[1], name='loss_sum')
self.loss = tfnn.reduce_mean(loss_sum, name='loss_mean')
if self.reg == 'l2':
with tfnn.name_scope('l2_reg'):
regularizers = 0
for W in self.Ws:
regularizers += tfnn.nn.l2_loss(W, name='l2_reg')
regularizers *= self.l2_placeholder
with tfnn.name_scope('l2_loss'):
self.loss += regularizers
tfnn.scalar_summary('loss', self.loss)
def _set_r2(self):
if isinstance(self.network, tfnn.RegNetwork):
with tfnn.name_scope('r2_score'):
self.ys_mean = ys_mean = tfnn.reduce_mean(self.network.target_placeholder,
reduction_indices=[0],
name='ys_mean')
self.ss_tot = ss_tot = tfnn.reduce_sum(
tfnn.square(self.network.target_placeholder - ys_mean),
reduction_indices=[0], name='total_sum_squares')
# ss_reg = np.sum(np.square(predictions-ys_mean), axis=0)
self.ss_res = ss_res = tfnn.reduce_sum(
tfnn.square(self.network.target_placeholder - self.network.predictions),
reduction_indices=[0], name='residual_sum_squares')
self.aaa = ss_res / ss_tot
self.r2 = tfnn.reduce_mean(
tfnn.sub(tfnn.ones_like(ss_res, dtype=tfnn.float32), (ss_res / ss_tot)),
name='coefficient_of_determination')
tfnn.scalar_summary('r2_score', self.r2)
def _init_loss(self):
with tfnn.name_scope('predictions'):
self.predictions = self.layers_results['final'][-1]
with tfnn.name_scope('loss'):
loss_square = tfnn.square(self.target_placeholder - self.predictions,
name='loss_square')
loss_sum = tfnn.reduce_sum(loss_square, reduction_indices=[1], name='loss_sum')
self.loss = tfnn.reduce_mean(loss_sum, name='loss_mean')
if self.reg == 'l2':
with tfnn.name_scope('l2_reg'):
regularizers = 0
for layer in self.layers_results['Layer'][1:]:
regularizers += tfnn.nn.l2_loss(layer.W, name='l2_reg')
regularizers *= self.l2_placeholder
with tfnn.name_scope('l2_loss'):
self.loss += regularizers
tfnn.scalar_summary('loss', self.loss)
def __init__(
self,
network,
):
self.network = network
if isinstance(self.network, tfnn.ClfNetwork):
with tfnn.name_scope('accuracy'):
with tfnn.name_scope('correct_prediction'):
correct_prediction = tfnn.equal(
tfnn.argmax(network.predictions, 1),
tfnn.argmax(network.target_placeholder, 1),
name='correct_prediction')
with tfnn.name_scope('accuracy'):
self.accuracy = tfnn.reduce_mean(tfnn.cast(
correct_prediction, tfnn.float32),
name='accuracy')
tfnn.scalar_summary('accuracy', self.accuracy)
elif isinstance(self.network, tfnn.RegNetwork):
self.first_time_lm = True
self.first_time_soc = True
with tfnn.name_scope('r2_score'):
with tfnn.name_scope('ys_mean'):
ys_mean = tfnn.reduce_mean(network.target_placeholder,
reduction_indices=[0],
name='ys_mean')
with tfnn.name_scope('total_sum_squares'):
ss_tot = tfnn.reduce_sum(
tfnn.square(network.target_placeholder - ys_mean),
reduction_indices=[0],
name='total_sum_squares')
# ss_reg = np.sum(np.square(predictions-ys_mean), axis=0)
with tfnn.name_scope('residual_sum_squares'):
ss_res = tfnn.reduce_sum(
tfnn.square(network.target_placeholder -
network.predictions),
reduction_indices=[0],
name='residual_sum_squares')
with tfnn.name_scope('coefficient_of_determination'):
self.r2_score = tfnn.sub(tfnn.constant(1, dtype=tfnn.float32),
(ss_res / ss_tot)[0],
name='coefficient_of_determination')
tfnn.scalar_summary('r2_score', self.r2_score)
def __init__(self, n_inputs, n_outputs, input_dtype, output_dtype, output_activator,
do_dropout, do_l2, seed=None):
self.normalizer = Normalizer()
self.n_inputs = n_inputs
self.n_outputs = n_outputs
self.input_dtype = input_dtype
self.output_dtype = output_dtype
self.output_activator = output_activator
if do_dropout and do_l2:
raise ValueError('Cannot do dropout and l2 at once. Choose only one of them.')
if do_dropout:
self.reg = 'dropout'
if do_l2:
self.reg = 'l2'
if (do_dropout is False) & (do_l2 is False):
self.reg = None
self.seed = seed
with tfnn.name_scope('inputs'):
self.data_placeholder = tfnn.placeholder(dtype=input_dtype, shape=[None, n_inputs], name='x_input')
self.target_placeholder = tfnn.placeholder(dtype=output_dtype, shape=[None, n_outputs], name='y_input')
if do_dropout:
self.keep_prob_placeholder = tfnn.placeholder(dtype=tfnn.float32)
tfnn.scalar_summary('dropout_keep_probability', self.keep_prob_placeholder)
if do_l2:
self.l2_placeholder = tfnn.placeholder(tfnn.float32)
tfnn.scalar_summary('l2_lambda', self.l2_placeholder)
self.layers_type = pd.Series([])
self.layers_output = pd.Series([])
self.layers_activated_output = pd.Series([])
self.layers_dropped_output = pd.Series([])
self.layers_final_output = pd.Series([])
self.Ws = pd.Series([])
self.bs = pd.Series([])
self.record_activators = pd.Series([])
self.record_neurons = []
self.last_layer_neurons = n_inputs
self.last_layer_outputs = self.data_placeholder
self.layer_number = 1
self.has_output_layer = False
self._is_output_layer = False
def _init_loss(self):
with tfnn.name_scope('predictions'):
self.predictions = self.layers_results['final'][-1]
with tfnn.name_scope('loss'):
loss_square = tfnn.square(self.target_placeholder -
self.predictions,
name='loss_square')
loss_sum = tfnn.reduce_sum(loss_square,
reduction_indices=[1],
name='loss_sum')
self.loss = tfnn.reduce_mean(loss_sum, name='loss_mean')
if self.reg == 'l2':
with tfnn.name_scope('l2_reg'):
regularizers = 0
for layer in self.layers_results['Layer'][1:]:
regularizers += tfnn.nn.l2_loss(layer.W, name='l2_reg')
regularizers *= self.l2_placeholder
with tfnn.name_scope('l2_loss'):
self.loss += regularizers
tfnn.scalar_summary('loss', self.loss)
def _init_loss(self):
with tfnn.name_scope('predictions'):
self.predictions = self.layers_final_output.iloc[-1] + 0
with tfnn.name_scope('loss'):
loss_square = tfnn.square(self.target_placeholder -
self.layers_final_output.iloc[-1],
name='loss_square')
loss_sum = tfnn.reduce_sum(loss_square,
reduction_indices=[1],
name='loss_sum')
self.loss = tfnn.reduce_mean(loss_sum, name='loss_mean')
if self.reg == 'l2':
with tfnn.name_scope('l2_reg'):
regularizers = 0
for W in self.Ws:
regularizers += tfnn.nn.l2_loss(W, name='l2_reg')
regularizers *= self.l2_placeholder
with tfnn.name_scope('l2_loss'):
self.loss += regularizers
tfnn.scalar_summary('loss', self.loss)
def __init__(self, n_inputs, n_outputs, input_dtype, output_dtype, output_activator,
do_dropout, do_l2, seed=None):
self.normalizer = Normalizer()
self.n_inputs = n_inputs
self.n_outputs = n_outputs
self.input_dtype = input_dtype
self.output_dtype = output_dtype
self.output_activator = output_activator
if do_dropout and do_l2:
raise ValueError('Cannot do dropout and l2 at once. Choose only one of them.')
if do_dropout:
self.reg = 'dropout'
if do_l2:
self.reg = 'l2'
if (do_dropout is False) & (do_l2 is False):
self.reg = None
self.seed = seed
with tfnn.name_scope('inputs'):
self.data_placeholder = tfnn.placeholder(dtype=input_dtype, shape=[None, n_inputs], name='x_input')
self.target_placeholder = tfnn.placeholder(dtype=output_dtype, shape=[None, n_outputs], name='y_input')
if do_dropout:
self.keep_prob_placeholder = tfnn.placeholder(dtype=tfnn.float32)
tfnn.scalar_summary('dropout_keep_probability', self.keep_prob_placeholder)
if do_l2:
self.l2_placeholder = tfnn.placeholder(tfnn.float32)
tfnn.scalar_summary('l2_lambda', self.l2_placeholder)
self.layers_output = pd.Series([])
self.layers_activated_output = pd.Series([])
self.layers_dropped_output = pd.Series([])
self.layers_final_output = pd.Series([])
self.Ws = pd.Series([])
self.bs = pd.Series([])
self.record_activators = pd.Series([])
self.record_neurons = []
self.last_layer_neurons = n_inputs
self.last_layer_outputs = self.data_placeholder
self.hidden_layer_number = 1
self.has_output_layer = False
self._is_output_layer = False
def _set_r2(self):
if isinstance(self.network, tfnn.RegNetwork):
with tfnn.name_scope('r2_score'):
self.ys_mean = ys_mean = tfnn.reduce_mean(
self.network.target_placeholder,
reduction_indices=[0],
name='ys_mean')
self.ss_tot = ss_tot = tfnn.reduce_sum(
tfnn.square(self.network.target_placeholder - ys_mean),
reduction_indices=[0],
name='total_sum_squares')
# ss_reg = np.sum(np.square(predictions-ys_mean), axis=0)
self.ss_res = ss_res = tfnn.reduce_sum(
tfnn.square(self.network.target_placeholder -
self.network.predictions),
reduction_indices=[0],
name='residual_sum_squares')
self.aaa = ss_res / ss_tot
self.r2 = tfnn.reduce_mean(tfnn.sub(
tfnn.ones_like(ss_res, dtype=tfnn.float32),
(ss_res / ss_tot)),
name='coefficient_of_determination')
tfnn.scalar_summary('r2_score', self.r2)
def _init_loss(self):
with tfnn.name_scope('predictions'):
if self.method == 'softmax':
self.predictions = tfnn.nn.softmax(
self.layers_final_output.iloc[-1], name='predictions')
elif self.method == 'sigmoid':
self.predictions = tfnn.nn.sigmoid(
self.layers_final_output.iloc[-1], name='predictions')
with tfnn.name_scope('loss'):
if self.method == 'softmax':
self.cross_entropy = tfnn.nn.softmax_cross_entropy_with_logits(
self.layers_final_output.iloc[-1],
self.target_placeholder,
name='xentropy')
elif self.method == 'sigmoid':
self.cross_entropy = tfnn.nn.sigmoid_cross_entropy_with_logits(
self.layers_final_output.iloc[-1],
self.target_placeholder,
name='xentropy')
else:
raise ValueError(
"method should be one of ['sparse_softmax', 'softmax', 'sigmoid']"
)
self.loss = tfnn.reduce_mean(self.cross_entropy,
name='xentropy_mean')
if self.reg == 'l2':
with tfnn.name_scope('l2_reg'):
regularizers = 0
for W in self.Ws:
regularizers += tfnn.nn.l2_loss(W, name='l2_reg')
regularizers *= self.l2_placeholder
with tfnn.name_scope('l2_loss'):
self.loss += regularizers
tfnn.scalar_summary('loss', self.loss)
def _set_confusion_metrics(self):
# from https://cloud.google.com/solutions/machine-learning-with-financial-time-series-data
# for onehot data
with tfnn.name_scope('f1_score'):
predictions = tfnn.argmax(self.network.predictions, 1)
actuals = tfnn.argmax(self.network.target_placeholder, 1)
ones_like_actuals = tfnn.ones_like(actuals)
zeros_like_actuals = tfnn.zeros_like(actuals)
ones_like_predictions = tfnn.ones_like(predictions)
zeros_like_predictions = tfnn.zeros_like(predictions)
tp = tfnn.reduce_sum(
tfnn.cast(
tfnn.logical_and(
tfnn.equal(actuals, ones_like_actuals),
tfnn.equal(predictions, ones_like_predictions)),
"float"))
tn = tfnn.reduce_sum(
tfnn.cast(
tfnn.logical_and(
tfnn.equal(actuals, zeros_like_actuals),
tfnn.equal(predictions, zeros_like_predictions)),
"float"))
fp = tfnn.reduce_sum(
tfnn.cast(
tfnn.logical_and(
tfnn.equal(actuals, zeros_like_actuals),
tfnn.equal(predictions, ones_like_predictions)),
"float"))
fn = tfnn.reduce_sum(
tfnn.cast(
tfnn.logical_and(
tfnn.equal(actuals, ones_like_actuals),
tfnn.equal(predictions, zeros_like_predictions)),
"float"))
self.recall = tp / (tp + fn)
self.precision = tp / (tp + fp)
self.f1 = tfnn.div(2 * (self.precision * self.recall),
(self.precision + self.recall),
name='f1_score')
tfnn.scalar_summary('f1_score', self.f1)
tfnn.scalar_summary('precision', self.precision)
tfnn.scalar_summary('recall', self.recall)
def _set_confusion_metrics(self):
# from https://cloud.google.com/solutions/machine-learning-with-financial-time-series-data
# for onehot data
with tfnn.name_scope('f1_score'):
predictions = tfnn.argmax(self.network.predictions, 1)
actuals = tfnn.argmax(self.network.target_placeholder, 1)
ones_like_actuals = tfnn.ones_like(actuals)
zeros_like_actuals = tfnn.zeros_like(actuals)
ones_like_predictions = tfnn.ones_like(predictions)
zeros_like_predictions = tfnn.zeros_like(predictions)
tp = tfnn.reduce_sum(
tfnn.cast(
tfnn.logical_and(
tfnn.equal(actuals, ones_like_actuals),
tfnn.equal(predictions, ones_like_predictions)
), "float"))
tn = tfnn.reduce_sum(
tfnn.cast(
tfnn.logical_and(
tfnn.equal(actuals, zeros_like_actuals),
tfnn.equal(predictions, zeros_like_predictions)
), "float"))
fp = tfnn.reduce_sum(
tfnn.cast(
tfnn.logical_and(
tfnn.equal(actuals, zeros_like_actuals),
tfnn.equal(predictions, ones_like_predictions)
), "float"))
fn = tfnn.reduce_sum(
tfnn.cast(
tfnn.logical_and(
tfnn.equal(actuals, ones_like_actuals),
tfnn.equal(predictions, zeros_like_predictions)
), "float"))
self.recall = tp / (tp + fn)
self.precision = tp / (tp + fp)
self.f1 = tfnn.div(2 * (self.precision * self.recall),
(self.precision + self.recall), name='f1_score')
tfnn.scalar_summary('f1_score', self.f1)
tfnn.scalar_summary('precision', self.precision)
tfnn.scalar_summary('recall', self.recall)