def process_validation_datasets_shortcuts(env_instance, set_of_kwargs,
taken_names):
validation_datasets = list()
if 'validation_datasets' in set_of_kwargs:
taken_names.extend(list(set_of_kwargs['validation_datasets'].keys()))
in_new_format = [
(v, k) for k, v in set_of_kwargs['validation_datasets'].items()
]
# validation_datasets.extend(set_of_kwargs['validation_datasets'])
validation_datasets.extend(in_new_format)
if 'validation_dataset_names' in set_of_kwargs:
for name in set_of_kwargs['validation_dataset_names']:
if name not in env_instance.datasets.keys():
raise InvalidArgumentError(
"Wrong value '%s' of variable '%s'\nAllowed values: '%s'" %
(name, "set_of_kwargs['validation_dataset_names']",
list(env_instance.datasets.keys())))
validation_datasets.append([env_instance.datasets[name], name])
if 'validation_dataset_texts' in set_of_kwargs:
for text in set_of_kwargs['validation_dataset_texts']:
key, value = process_input_text_dataset(text, taken_names)
taken_names.append(key)
validation_datasets.append([value, key])
if 'validation_dataset_filenames' in set_of_kwargs:
for filename in set_of_kwargs['validation_dataset_filenames']:
key, value = env_instance.process_dataset_filename(filename)
taken_names.append(key)
validation_datasets.append([value, key])
_ = remove_keys_from_dictionary(set_of_kwargs, [
'validation_datasets', 'validation_dataset_names',
'validation_dataset_texts', 'validation_dataset_filenames'
])
return validation_datasets
def process_shortcut_with_dataset_names(env_instance, set_of_kwargs, key):
all_names_are_present = True
for name in set_of_kwargs[key]:
all_names_are_present = all_names_are_present and (name in list(
env_instance.datasets.keys()))
if not all_names_are_present:
raise InvalidArgumentError(
"Wrong value '%s' of variable '%s'\nAllowed values: '%s'" %
(set_of_kwargs[key], "set_of_kwargs[%s]" % key,
list(env_instance.datasets.keys())), set_of_kwargs[key],
"set_of_kwargs[%s]" % key, list(env_instance.datasets.keys()))
return [[env_instance.datasets[name], name]
for name in list(env_instance.datasets.keys())]
def process_single_dataset_shortcut(
env_instance,
set_of_kwargs,
taken_names,
key
):
if key == 'train_dataset':
if 'train_dataset' in set_of_kwargs:
taken_names.extend(list(set_of_kwargs['train_dataset'].keys()))
ret = [(v, k) for k, v in set_of_kwargs['train_dataset'].items()][0]
elif 'train_dataset_name' in set_of_kwargs:
if set_of_kwargs['train_dataset_name'] not in env_instance.datasets.keys():
raise InvalidArgumentError("Wrong value '%s' of variable '%s'\nAllowed values: '%s'" %
(set_of_kwargs['train_dataset_name'], "set_of_kwargs['train_dataset_name']",
list(env_instance.datasets.keys())),
set_of_kwargs['train_dataset_name'],
"set_of_kwargs['train_dataset_name']",
list(env_instance.datasets.keys()))
ret = [env_instance.datasets[set_of_kwargs['train_dataset_name']], set_of_kwargs['train_dataset_name']]
elif 'train_dataset_text' in set_of_kwargs:
key, value = process_input_text_dataset(set_of_kwargs['train_dataset_text'], taken_names)
taken_names.append(key)
ret = [value, key]
elif 'train_dataset_filename' in set_of_kwargs:
key, value = process_dataset_filename(env_instance, set_of_kwargs['train_dataset_filename'])
taken_names.append(key)
ret = [value, key]
else:
ret = None
_ = remove_keys_from_dictionary(
set_of_kwargs, ['train_dataset', 'train_dataset_name', 'train_dataset_text', 'train_dataset_filename'])
return ret
if key == 'train_datasets':
return process_standard_combination_of_list_of_dataset_abbreviations(
env_instance, set_of_kwargs, taken_names,
['train_datasets', 'train_dataset_names', 'train_dataset_texts', 'train_dataset_filenames'])
if key == 'opt_inf_train_datasets':
return process_standard_combination_of_list_of_dataset_abbreviations(
env_instance, set_of_kwargs, taken_names,
['opt_inf_train_datasets', 'opt_inf_train_dataset_names',
'opt_inf_train_dataset_texts', 'opt_inf_train_dataset_filenames']
)
if key == 'opt_inf_validation_datasets':
return process_standard_combination_of_list_of_dataset_abbreviations(
env_instance, set_of_kwargs, taken_names,
['opt_inf_validation_datasets', 'opt_inf_validation_dataset_names',
'opt_inf_validation_dataset_texts', 'opt_inf_validation_dataset_filenames']
)
def compute_metrics(metrics,
predictions=None,
labels=None,
loss=None,
keep_first_dim=False):
# print("(tensors.compute_metrics)predictions.shape:", predictions.get_shape().as_list())
# print("(tensors.compute_metrics)labels.shape:", labels.get_shape().as_list())
# print("(tensors.compute_metrics)loss.shape:", loss.get_shape().as_list())
with tf.name_scope('compute_metrics'):
res = dict()
if 'loss' in metrics:
l = loss_tensor(predictions=predictions,
labels=labels,
keep_first_dim=keep_first_dim)
res['loss'] = l
else:
l = None
if 'bpc' in metrics:
if loss is not None:
bpc = bpc_tensor(loss=loss)
res['bpc'] = bpc
elif l is not None:
bpc = bpc_tensor(loss=l)
res['bpc'] = bpc
elif predictions is not None and labels is not None:
bpc = bpc_tensor(
loss=loss_tensor(predictions=predictions,
labels=labels,
keep_first_dim=keep_first_dim))
res['bpc'] = bpc
else:
print('loss:', loss)
print('metrics:', metrics)
print('predictions:', predictions)
print('labels:', labels)
raise InvalidArgumentError(
'Could not build bpc graph. Not enough arguments were provided.',
[metrics, predictions, labels, loss],
['metrics', 'predictions', 'labels', 'loss'],
'At least loss or predictions and labels has to be not None'
)
if 'accuracy' in metrics:
accuracy = accuracy_tensor(predictions=predictions,
labels=labels,
keep_first_dim=keep_first_dim)
res['accuracy'] = accuracy
if 'perplexity' in metrics:
perplexity = perplexity_tensor(probabilities=predictions,
keep_first_dim=keep_first_dim)
res['perplexity'] = perplexity
return res
yscale = sys.argv[5]
limit = sys.argv[6]
if limit == 'None':
limit = None
else:
limit = int(limit)
shift = sys.argv[7]
if limit == 'None':
shift = None
else:
shift = int(shift)
plot_data = sys.argv[8:]
if len(plot_data) % 2 != 0:
raise InvalidArgumentError(
"Every file has to be provided with line name (number of arguments has to be even)",
[plot_data, len(plot_data)],
'plot_data',
"len(plot_data) % 2 == 0"
)
line_names = [plot_data[2*i] for i in range(len(plot_data) // 2)]
file_names = [plot_data[2*i+1] for i in range(len(plot_data) // 2)]
# print(line_names)
data = list()
for line_name, file_name in zip(line_names, file_names):
with open(file_name, 'r') as f:
lines = f.read().split('\n')
lines = lines[:limit]
while len(lines) > 0 and len(lines[-1]) == 0:
lines = lines[:-1]
def __init__(
self,
batch_size=64,
num_layers=2,
num_nodes=None,
num_output_layers=1,
num_output_nodes=None,
vocabulary_size=None,
embedding_size=128,
num_unrollings=10,
init_parameter=3.,
num_gpus=1,
regularization_rate=.000006,
additional_metrics=None,
regime='autonomous_training',
going_to_limit_memory=False,
optimizer='adam',
rho=0.95, # used only for adadelta
decay=0.9, # used for rmsprop
):
"""4 regimes are possible: autonomous_training, inference, training_with_meta_optimizer, optimizer_training"""
if num_nodes is None:
num_nodes = [112, 113]
if num_output_nodes is None:
num_output_nodes = list()
if additional_metrics is None:
additional_metrics = list()
self._batch_size = batch_size
self._num_layers = num_layers
self._num_nodes = num_nodes
self._vocabulary_size = vocabulary_size
self._embedding_size = embedding_size
self._num_output_layers = num_output_layers
self._num_output_nodes = num_output_nodes
self._num_unrollings = num_unrollings
self._init_parameter = init_parameter
self._regularization_rate = regularization_rate
self._additional_metrics = additional_metrics
self._optimizer = optimizer
self._rho = rho
self._decay = decay
self._hooks = dict(
inputs=None,
labels=None,
labels_prepared=None,
train_op=None,
learning_rate=None,
momentum=None,
loss=None,
predictions=None,
validation_inputs=None,
validation_labels=None,
validation_labels_prepared=None,
validation_predictions=None,
reset_validation_state=None,
reset_pupil_train_state=None,
randomize_sample_state=None,
dropout=None,
saver=None)
for add_metric in self._additional_metrics:
self._hooks[add_metric] = None
self._hooks['validation_' + add_metric] = None
if not going_to_limit_memory:
gpu_names = get_available_gpus()
self._gpu_names = ['/gpu:%s' % i for i in range(len(gpu_names))]
else:
self._gpu_names = ['/gpu:%s' % i for i in range(num_gpus)]
num_available_gpus = len(self._gpu_names)
num_gpus, self._batch_sizes_on_gpus = get_num_gpus_and_bs_on_gpus(
self._batch_size, num_gpus, num_available_gpus)
self._num_gpus = num_gpus
self._vec_dim = self._vocabulary_size
if self._num_gpus == 1:
self._base_device = '/gpu:0'
else:
self._base_device = '/cpu:0'
self._dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')
self._hooks['dropout'] = self._dropout_keep_prob
self._applicable_trainable = dict()
self._train_storage = dict()
self._inference_storage = dict()
self._train_inputs_and_labels_placeholders = dict()
self._autonomous_train_specific_placeholders = dict()
self._inference_placeholders = dict()
if regime == 'autonomous_training':
self._add_trainable_variables()
# self._add_train_storage()
self._add_autonomous_training_specific_placeholders()
self._add_train_inputs_and_labels_placeholders()
self._train_graph()
self._validation_graph()
elif regime == 'inference':
self._add_trainable_variables()
self._validation_graph()
elif regime == 'training_with_meta_optimizer':
self._add_trainable_variables()
self._add_train_storage()
self._add_train_inputs_and_labels_placeholders()
self._hooks['reset_pupil_train_state'] = tf.group(*self.reset_self_train_storage())
self._validation_graph()
elif regime == 'optimizer_training':
self._add_trainable_variables()
self._add_train_storage()
self._add_train_inputs_and_labels_placeholders()
self._hooks['reset_pupil_train_state'] = tf.group(*self.reset_self_train_storage())
self._validation_graph()
else:
raise InvalidArgumentError(
'Not allowed regime',
regime,
'regime',
['autonomous_training', 'inference', 'training_with_meta_optimizer', 'optimizer_training']
)
def loss_and_opt_ins(
self, inputs, labels, storage, opt_ins=None, trainable_variables=None, name_scope='pupil_loss'):
"""Args:
either trainable_variables or opt_ins have to be provided
optimizer_ins is a dictionary which keys are layer names and values are dictionaries with their parameters
('matrix' and optionally 'bias') (meaning tf.Variable instances holding their saved values) and 'o' ansd 's'
vectors. 'o' - vector is an output of previous layer (input for linear projection) and 's' is the result of
linear projection performed using layer weights. 'o', 's', 'matrix', 'bias' - can be stacked if several
exercises are processed"""
optimizer_ins = dict()
with tf.name_scope(name_scope):
saved_states = storage['states']
if opt_ins is not None:
trainable = self._extract_trainable_from_opt_ins(opt_ins)
elif trainable_variables is not None:
trainable = trainable_variables
else:
raise InvalidArgumentError(
'At least one of arguments opt_ins or trainable_variables have to be provided',
(None, None),
('opt_ins', 'trainable_variables'),
'opt_ins in optimizer_ins format and trainable_variables structure is explained in loss_and_opt_ins'
'implementation'
)
embedding_matrix = trainable['embedding_matrix']
lstm_matrices = trainable['lstm_matrices']
lstm_biases = trainable['lstm_biases']
output_matrices = trainable['output_matrices']
output_biases = trainable['output_biases']
# with tf.device('/cpu:0'):
# embedding_matrix = tf.Print(
# embedding_matrix, [embedding_matrix], message="\nembedding_matrix: ", summarize=10)
# n_lstm_matrices = list()
# for lstm_matrix in lstm_matrices:
# n_lstm_matrices.append(tf.Print(
# lstm_matrix, [lstm_matrix], message="lstm_matrix: ", summarize=10))
# n_output_matrices = list()
# for output_matrix in output_matrices:
# n_output_matrices.append(tf.Print(
# output_matrix, [output_matrix], message="output_matrix: ", summarize=10))
# lstm_matrices = n_lstm_matrices
# output_matrices = n_output_matrices
# n_saved_states = list()
# for layer_saved_states in saved_states:
# n_saved_states.append([
# tf.Print(layer_saved_states[0], [layer_saved_states[0]], message="h: ", summarize=10),
# tf.Print(layer_saved_states[1], [layer_saved_states[1]], message="c: ", summarize=10)
# ])
# saved_states = n_saved_states
inputs, labels = self._prepare_inputs_and_labels(inputs, labels)
embeddings, opt_ins = self._embed(inputs, embedding_matrix)
optimizer_ins.update(opt_ins)
rnn_outputs, new_states, opt_ins = self._rnn_module(
embeddings, saved_states, lstm_matrices, lstm_biases)
optimizer_ins.update(opt_ins)
logits, opt_ins = self._output_module(rnn_outputs, output_matrices, output_biases)
optimizer_ins.update(opt_ins)
new_storage = dict(
states=new_states
)
labels_shape = labels.get_shape().as_list()
logits = tf.reshape(logits, labels_shape),
predictions = tf.nn.softmax(logits)
# print('(LstmForMeta.loss_and_opt_ins)labels_shape:', labels.get_shape().as_list())
# print('(LstmForMeta.loss_and_opt_ins)logits_shape:', logits.get_shape().as_list())
sce = tf.nn.softmax_cross_entropy_with_logits(
labels=tf.reshape(labels, [-1, labels_shape[-1]]),
logits=tf.reshape(logits, [-1, labels_shape[-1]])
)
loss = tf.reduce_mean(
tf.reshape(sce, labels_shape[:-1]),
axis=[-1, -2]
)
optimizer_ins = self._acomplish_optimizer_ins(optimizer_ins, trainable)
return loss, optimizer_ins, new_storage, predictions, labels
def identity_tensor(**kwargs):
if len(kwargs) > 1:
raise InvalidArgumentError('kwargs should not contain 1 entry', kwargs, 'kwargs', 'len(kwargs)=1')
for value in kwargs.values():
return value
def loss_and_opt_ins(self,
inputs,
labels,
pgeps,
opt_ins=None,
trainable_variables=None,
name_scope='pupil_loss'):
"""Args:
either trainable_variables or opt_ins have to be provided
optimizer_ins is a dictionary which keys are layer names and values are dictionaries with their parameters
('matrix' and optionally 'bias') (meaning tf.Variable instances holding their saved values) and 'o' ansd 's'
vectors. 'o' - vector is an output of previous layer (input for linear projection) and 's' is the result of
linear projection performed using layer weights. 'o', 's', 'matrix', 'bias' - can be stacked if several
exercises are processed"""
with tf.name_scope(name_scope):
if opt_ins is not None:
trainable = self._extract_trainable_from_opt_ins(opt_ins)
elif trainable_variables is not None:
trainable = trainable_variables
else:
raise InvalidArgumentError(
'At least one of arguments opt_ins or trainable_variables have to be provided',
(None, None), ('opt_ins', 'trainable_variables'),
'opt_ins in optimizer_ins format and trainable_variables structure is explained in loss_and_opt_ins'
'implementation')
matrices = trainable['matrices']
biases = trainable['biases']
logits, opt_ins = self._mlp(inputs, matrices, biases)
self._logits = logits
# with tf.device('/cpu:0'):
# logits = tf.Print(
# logits,
# [logits],
# message="\n(MlpForMeta.loss_and_opt_ins)logits:",
# summarize=320
# )
# logits = tf.Print(
# logits,
# [inputs],
# message="\n(MlpForMeta.loss_and_opt_ins)inputs:",
# summarize=320
# )
predictions = tf.nn.softmax(logits)
# with tf.device('/cpu:0'):
# predictions = tf.Print(
# predictions,
# [predictions],
# message="\n(MlpForMeta.loss_and_opt_ins)predictions:",
# summarize=320
# )
# print('(LstmForMeta.loss_and_opt_ins)labels_shape:', labels.get_shape().as_list())
# print('(LstmForMeta.loss_and_opt_ins)logits_shape:', logits.get_shape().as_list())
labels = tf.one_hot(labels, self._num_classes)
labels_shape = [
-1 if a is None else a for a in labels.get_shape().as_list()
]
sce = tf.nn.softmax_cross_entropy_with_logits(labels=labels,
logits=logits)
sce = tf.reshape(sce, labels_shape[:-1])
loss = tf.reduce_mean(sce, axis=[-1])
optimizer_ins = self._acomplish_optimizer_ins(opt_ins, trainable)
return loss, optimizer_ins, dict(), predictions, labels
def __init__(
self,
batch_size=32,
num_layers=2,
num_hidden_nodes=None, # a list which length is by 1 less than num_layers
init_parameter=3.,
regularization_rate=.000006,
additional_metrics=None,
input_shape=None,
num_classes=None,
regime='autonomous_training',
optimizer='adam',
rho=0.95, # used for adadelta
decay=0.9, # used for rmsprop
):
if num_hidden_nodes is None:
num_hidden_nodes = [1000, 1000]
self._batch_size = batch_size
self._num_layers = num_layers
self._num_hidden_nodes = num_hidden_nodes
self._init_parameter = init_parameter
self._regularization_rate = regularization_rate
self._input_shape = input_shape
self._input_size = cumulative_mul(input_shape, 1)
self._input_ndim = len(self._input_shape)
self._num_classes = num_classes
self._additional_metrics = additional_metrics
self._optimizer = optimizer
self._rho = rho
self._decay = decay
self._hooks = dict(inputs=None,
labels=None,
labels_prepared=None,
train_op=None,
learning_rate=None,
momentum=None,
loss=None,
predictions=None,
validation_inputs=None,
validation_labels=None,
validation_predictions=None,
dropout=None,
saver=None)
for add_metric in self._additional_metrics:
self._hooks[add_metric] = None
self._hooks['validation_' + add_metric] = None
self._dropout_keep_prob = tf.placeholder(tf.float32,
name='dropout_keep_prob')
self._hooks['dropout'] = self._dropout_keep_prob
self._trainable = dict()
self._inputs_and_labels_placeholders = dict()
self._autonomous_train_specific_placeholders = dict()
self._add_trainable_variables()
self._add_inputs_and_labels()
loss, optimizer_ins, _, predictions, labels = self.loss_and_opt_ins_for_inference(
)
self._add_metrics_and_hooks(loss, predictions, labels)
if regime == 'autonomous_training':
self._add_autonomous_training_specific_placeholders()
self._add_train_op(loss)
elif regime == 'inference':
pass
elif regime == 'training_with_meta_optimizer':
pass
elif regime == 'optimizer_training':
pass
else:
raise InvalidArgumentError(
'Not allowed regime', regime, 'regime', [
'autonomous_training', 'inference',
'training_with_meta_optimizer', 'optimizer_training'
])
yscale = sys.argv[5]
limit = sys.argv[6]
if limit == 'None':
limit = None
else:
limit = int(limit)
shift = sys.argv[7]
if limit == 'None':
shift = None
else:
shift = int(shift)
labels_and_files = sys.argv[8:]
if len(labels_and_files) % 2 != 0:
raise InvalidArgumentError(
"Every file has to be provided with line name (number of arguments has to be even)",
[labels_and_files, len(labels_and_files)],
'labels_and_files',
"len(labels_and_files) % 2 == 0"
)
line_names = [labels_and_files[2 * i] for i in range(len(labels_and_files) // 2)]
file_names = [labels_and_files[2 * i + 1] for i in range(len(labels_and_files) // 2)]
# print(line_names)
data = list()
for line_name, file_name in zip(line_names, file_names):
with open(file_name, 'r') as f:
lines = f.read().split('\n')
lines = lines[:limit]
while len(lines) > 0 and len(lines[-1]) == 0:
lines = lines[:-1]