def process_validation_datasets_shortcuts(env_instance, set_of_kwargs, taken_names): validation_datasets = list() if 'validation_datasets' in set_of_kwargs: taken_names.extend(list(set_of_kwargs['validation_datasets'].keys())) in_new_format = [ (v, k) for k, v in set_of_kwargs['validation_datasets'].items() ] # validation_datasets.extend(set_of_kwargs['validation_datasets']) validation_datasets.extend(in_new_format) if 'validation_dataset_names' in set_of_kwargs: for name in set_of_kwargs['validation_dataset_names']: if name not in env_instance.datasets.keys(): raise InvalidArgumentError( "Wrong value '%s' of variable '%s'\nAllowed values: '%s'" % (name, "set_of_kwargs['validation_dataset_names']", list(env_instance.datasets.keys()))) validation_datasets.append([env_instance.datasets[name], name]) if 'validation_dataset_texts' in set_of_kwargs: for text in set_of_kwargs['validation_dataset_texts']: key, value = process_input_text_dataset(text, taken_names) taken_names.append(key) validation_datasets.append([value, key]) if 'validation_dataset_filenames' in set_of_kwargs: for filename in set_of_kwargs['validation_dataset_filenames']: key, value = env_instance.process_dataset_filename(filename) taken_names.append(key) validation_datasets.append([value, key]) _ = remove_keys_from_dictionary(set_of_kwargs, [ 'validation_datasets', 'validation_dataset_names', 'validation_dataset_texts', 'validation_dataset_filenames' ]) return validation_datasets
def process_shortcut_with_dataset_names(env_instance, set_of_kwargs, key): all_names_are_present = True for name in set_of_kwargs[key]: all_names_are_present = all_names_are_present and (name in list( env_instance.datasets.keys())) if not all_names_are_present: raise InvalidArgumentError( "Wrong value '%s' of variable '%s'\nAllowed values: '%s'" % (set_of_kwargs[key], "set_of_kwargs[%s]" % key, list(env_instance.datasets.keys())), set_of_kwargs[key], "set_of_kwargs[%s]" % key, list(env_instance.datasets.keys())) return [[env_instance.datasets[name], name] for name in list(env_instance.datasets.keys())]
def process_single_dataset_shortcut( env_instance, set_of_kwargs, taken_names, key ): if key == 'train_dataset': if 'train_dataset' in set_of_kwargs: taken_names.extend(list(set_of_kwargs['train_dataset'].keys())) ret = [(v, k) for k, v in set_of_kwargs['train_dataset'].items()][0] elif 'train_dataset_name' in set_of_kwargs: if set_of_kwargs['train_dataset_name'] not in env_instance.datasets.keys(): raise InvalidArgumentError("Wrong value '%s' of variable '%s'\nAllowed values: '%s'" % (set_of_kwargs['train_dataset_name'], "set_of_kwargs['train_dataset_name']", list(env_instance.datasets.keys())), set_of_kwargs['train_dataset_name'], "set_of_kwargs['train_dataset_name']", list(env_instance.datasets.keys())) ret = [env_instance.datasets[set_of_kwargs['train_dataset_name']], set_of_kwargs['train_dataset_name']] elif 'train_dataset_text' in set_of_kwargs: key, value = process_input_text_dataset(set_of_kwargs['train_dataset_text'], taken_names) taken_names.append(key) ret = [value, key] elif 'train_dataset_filename' in set_of_kwargs: key, value = process_dataset_filename(env_instance, set_of_kwargs['train_dataset_filename']) taken_names.append(key) ret = [value, key] else: ret = None _ = remove_keys_from_dictionary( set_of_kwargs, ['train_dataset', 'train_dataset_name', 'train_dataset_text', 'train_dataset_filename']) return ret if key == 'train_datasets': return process_standard_combination_of_list_of_dataset_abbreviations( env_instance, set_of_kwargs, taken_names, ['train_datasets', 'train_dataset_names', 'train_dataset_texts', 'train_dataset_filenames']) if key == 'opt_inf_train_datasets': return process_standard_combination_of_list_of_dataset_abbreviations( env_instance, set_of_kwargs, taken_names, ['opt_inf_train_datasets', 'opt_inf_train_dataset_names', 'opt_inf_train_dataset_texts', 'opt_inf_train_dataset_filenames'] ) if key == 'opt_inf_validation_datasets': return process_standard_combination_of_list_of_dataset_abbreviations( env_instance, set_of_kwargs, taken_names, ['opt_inf_validation_datasets', 'opt_inf_validation_dataset_names', 'opt_inf_validation_dataset_texts', 'opt_inf_validation_dataset_filenames'] )
def compute_metrics(metrics, predictions=None, labels=None, loss=None, keep_first_dim=False): # print("(tensors.compute_metrics)predictions.shape:", predictions.get_shape().as_list()) # print("(tensors.compute_metrics)labels.shape:", labels.get_shape().as_list()) # print("(tensors.compute_metrics)loss.shape:", loss.get_shape().as_list()) with tf.name_scope('compute_metrics'): res = dict() if 'loss' in metrics: l = loss_tensor(predictions=predictions, labels=labels, keep_first_dim=keep_first_dim) res['loss'] = l else: l = None if 'bpc' in metrics: if loss is not None: bpc = bpc_tensor(loss=loss) res['bpc'] = bpc elif l is not None: bpc = bpc_tensor(loss=l) res['bpc'] = bpc elif predictions is not None and labels is not None: bpc = bpc_tensor( loss=loss_tensor(predictions=predictions, labels=labels, keep_first_dim=keep_first_dim)) res['bpc'] = bpc else: print('loss:', loss) print('metrics:', metrics) print('predictions:', predictions) print('labels:', labels) raise InvalidArgumentError( 'Could not build bpc graph. Not enough arguments were provided.', [metrics, predictions, labels, loss], ['metrics', 'predictions', 'labels', 'loss'], 'At least loss or predictions and labels has to be not None' ) if 'accuracy' in metrics: accuracy = accuracy_tensor(predictions=predictions, labels=labels, keep_first_dim=keep_first_dim) res['accuracy'] = accuracy if 'perplexity' in metrics: perplexity = perplexity_tensor(probabilities=predictions, keep_first_dim=keep_first_dim) res['perplexity'] = perplexity return res
yscale = sys.argv[5] limit = sys.argv[6] if limit == 'None': limit = None else: limit = int(limit) shift = sys.argv[7] if limit == 'None': shift = None else: shift = int(shift) plot_data = sys.argv[8:] if len(plot_data) % 2 != 0: raise InvalidArgumentError( "Every file has to be provided with line name (number of arguments has to be even)", [plot_data, len(plot_data)], 'plot_data', "len(plot_data) % 2 == 0" ) line_names = [plot_data[2*i] for i in range(len(plot_data) // 2)] file_names = [plot_data[2*i+1] for i in range(len(plot_data) // 2)] # print(line_names) data = list() for line_name, file_name in zip(line_names, file_names): with open(file_name, 'r') as f: lines = f.read().split('\n') lines = lines[:limit] while len(lines) > 0 and len(lines[-1]) == 0: lines = lines[:-1]
def __init__( self, batch_size=64, num_layers=2, num_nodes=None, num_output_layers=1, num_output_nodes=None, vocabulary_size=None, embedding_size=128, num_unrollings=10, init_parameter=3., num_gpus=1, regularization_rate=.000006, additional_metrics=None, regime='autonomous_training', going_to_limit_memory=False, optimizer='adam', rho=0.95, # used only for adadelta decay=0.9, # used for rmsprop ): """4 regimes are possible: autonomous_training, inference, training_with_meta_optimizer, optimizer_training""" if num_nodes is None: num_nodes = [112, 113] if num_output_nodes is None: num_output_nodes = list() if additional_metrics is None: additional_metrics = list() self._batch_size = batch_size self._num_layers = num_layers self._num_nodes = num_nodes self._vocabulary_size = vocabulary_size self._embedding_size = embedding_size self._num_output_layers = num_output_layers self._num_output_nodes = num_output_nodes self._num_unrollings = num_unrollings self._init_parameter = init_parameter self._regularization_rate = regularization_rate self._additional_metrics = additional_metrics self._optimizer = optimizer self._rho = rho self._decay = decay self._hooks = dict( inputs=None, labels=None, labels_prepared=None, train_op=None, learning_rate=None, momentum=None, loss=None, predictions=None, validation_inputs=None, validation_labels=None, validation_labels_prepared=None, validation_predictions=None, reset_validation_state=None, reset_pupil_train_state=None, randomize_sample_state=None, dropout=None, saver=None) for add_metric in self._additional_metrics: self._hooks[add_metric] = None self._hooks['validation_' + add_metric] = None if not going_to_limit_memory: gpu_names = get_available_gpus() self._gpu_names = ['/gpu:%s' % i for i in range(len(gpu_names))] else: self._gpu_names = ['/gpu:%s' % i for i in range(num_gpus)] num_available_gpus = len(self._gpu_names) num_gpus, self._batch_sizes_on_gpus = get_num_gpus_and_bs_on_gpus( self._batch_size, num_gpus, num_available_gpus) self._num_gpus = num_gpus self._vec_dim = self._vocabulary_size if self._num_gpus == 1: self._base_device = '/gpu:0' else: self._base_device = '/cpu:0' self._dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob') self._hooks['dropout'] = self._dropout_keep_prob self._applicable_trainable = dict() self._train_storage = dict() self._inference_storage = dict() self._train_inputs_and_labels_placeholders = dict() self._autonomous_train_specific_placeholders = dict() self._inference_placeholders = dict() if regime == 'autonomous_training': self._add_trainable_variables() # self._add_train_storage() self._add_autonomous_training_specific_placeholders() self._add_train_inputs_and_labels_placeholders() self._train_graph() self._validation_graph() elif regime == 'inference': self._add_trainable_variables() self._validation_graph() elif regime == 'training_with_meta_optimizer': self._add_trainable_variables() self._add_train_storage() self._add_train_inputs_and_labels_placeholders() self._hooks['reset_pupil_train_state'] = tf.group(*self.reset_self_train_storage()) self._validation_graph() elif regime == 'optimizer_training': self._add_trainable_variables() self._add_train_storage() self._add_train_inputs_and_labels_placeholders() self._hooks['reset_pupil_train_state'] = tf.group(*self.reset_self_train_storage()) self._validation_graph() else: raise InvalidArgumentError( 'Not allowed regime', regime, 'regime', ['autonomous_training', 'inference', 'training_with_meta_optimizer', 'optimizer_training'] )
def loss_and_opt_ins( self, inputs, labels, storage, opt_ins=None, trainable_variables=None, name_scope='pupil_loss'): """Args: either trainable_variables or opt_ins have to be provided optimizer_ins is a dictionary which keys are layer names and values are dictionaries with their parameters ('matrix' and optionally 'bias') (meaning tf.Variable instances holding their saved values) and 'o' ansd 's' vectors. 'o' - vector is an output of previous layer (input for linear projection) and 's' is the result of linear projection performed using layer weights. 'o', 's', 'matrix', 'bias' - can be stacked if several exercises are processed""" optimizer_ins = dict() with tf.name_scope(name_scope): saved_states = storage['states'] if opt_ins is not None: trainable = self._extract_trainable_from_opt_ins(opt_ins) elif trainable_variables is not None: trainable = trainable_variables else: raise InvalidArgumentError( 'At least one of arguments opt_ins or trainable_variables have to be provided', (None, None), ('opt_ins', 'trainable_variables'), 'opt_ins in optimizer_ins format and trainable_variables structure is explained in loss_and_opt_ins' 'implementation' ) embedding_matrix = trainable['embedding_matrix'] lstm_matrices = trainable['lstm_matrices'] lstm_biases = trainable['lstm_biases'] output_matrices = trainable['output_matrices'] output_biases = trainable['output_biases'] # with tf.device('/cpu:0'): # embedding_matrix = tf.Print( # embedding_matrix, [embedding_matrix], message="\nembedding_matrix: ", summarize=10) # n_lstm_matrices = list() # for lstm_matrix in lstm_matrices: # n_lstm_matrices.append(tf.Print( # lstm_matrix, [lstm_matrix], message="lstm_matrix: ", summarize=10)) # n_output_matrices = list() # for output_matrix in output_matrices: # n_output_matrices.append(tf.Print( # output_matrix, [output_matrix], message="output_matrix: ", summarize=10)) # lstm_matrices = n_lstm_matrices # output_matrices = n_output_matrices # n_saved_states = list() # for layer_saved_states in saved_states: # n_saved_states.append([ # tf.Print(layer_saved_states[0], [layer_saved_states[0]], message="h: ", summarize=10), # tf.Print(layer_saved_states[1], [layer_saved_states[1]], message="c: ", summarize=10) # ]) # saved_states = n_saved_states inputs, labels = self._prepare_inputs_and_labels(inputs, labels) embeddings, opt_ins = self._embed(inputs, embedding_matrix) optimizer_ins.update(opt_ins) rnn_outputs, new_states, opt_ins = self._rnn_module( embeddings, saved_states, lstm_matrices, lstm_biases) optimizer_ins.update(opt_ins) logits, opt_ins = self._output_module(rnn_outputs, output_matrices, output_biases) optimizer_ins.update(opt_ins) new_storage = dict( states=new_states ) labels_shape = labels.get_shape().as_list() logits = tf.reshape(logits, labels_shape), predictions = tf.nn.softmax(logits) # print('(LstmForMeta.loss_and_opt_ins)labels_shape:', labels.get_shape().as_list()) # print('(LstmForMeta.loss_and_opt_ins)logits_shape:', logits.get_shape().as_list()) sce = tf.nn.softmax_cross_entropy_with_logits( labels=tf.reshape(labels, [-1, labels_shape[-1]]), logits=tf.reshape(logits, [-1, labels_shape[-1]]) ) loss = tf.reduce_mean( tf.reshape(sce, labels_shape[:-1]), axis=[-1, -2] ) optimizer_ins = self._acomplish_optimizer_ins(optimizer_ins, trainable) return loss, optimizer_ins, new_storage, predictions, labels
def identity_tensor(**kwargs): if len(kwargs) > 1: raise InvalidArgumentError('kwargs should not contain 1 entry', kwargs, 'kwargs', 'len(kwargs)=1') for value in kwargs.values(): return value
def loss_and_opt_ins(self, inputs, labels, pgeps, opt_ins=None, trainable_variables=None, name_scope='pupil_loss'): """Args: either trainable_variables or opt_ins have to be provided optimizer_ins is a dictionary which keys are layer names and values are dictionaries with their parameters ('matrix' and optionally 'bias') (meaning tf.Variable instances holding their saved values) and 'o' ansd 's' vectors. 'o' - vector is an output of previous layer (input for linear projection) and 's' is the result of linear projection performed using layer weights. 'o', 's', 'matrix', 'bias' - can be stacked if several exercises are processed""" with tf.name_scope(name_scope): if opt_ins is not None: trainable = self._extract_trainable_from_opt_ins(opt_ins) elif trainable_variables is not None: trainable = trainable_variables else: raise InvalidArgumentError( 'At least one of arguments opt_ins or trainable_variables have to be provided', (None, None), ('opt_ins', 'trainable_variables'), 'opt_ins in optimizer_ins format and trainable_variables structure is explained in loss_and_opt_ins' 'implementation') matrices = trainable['matrices'] biases = trainable['biases'] logits, opt_ins = self._mlp(inputs, matrices, biases) self._logits = logits # with tf.device('/cpu:0'): # logits = tf.Print( # logits, # [logits], # message="\n(MlpForMeta.loss_and_opt_ins)logits:", # summarize=320 # ) # logits = tf.Print( # logits, # [inputs], # message="\n(MlpForMeta.loss_and_opt_ins)inputs:", # summarize=320 # ) predictions = tf.nn.softmax(logits) # with tf.device('/cpu:0'): # predictions = tf.Print( # predictions, # [predictions], # message="\n(MlpForMeta.loss_and_opt_ins)predictions:", # summarize=320 # ) # print('(LstmForMeta.loss_and_opt_ins)labels_shape:', labels.get_shape().as_list()) # print('(LstmForMeta.loss_and_opt_ins)logits_shape:', logits.get_shape().as_list()) labels = tf.one_hot(labels, self._num_classes) labels_shape = [ -1 if a is None else a for a in labels.get_shape().as_list() ] sce = tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits) sce = tf.reshape(sce, labels_shape[:-1]) loss = tf.reduce_mean(sce, axis=[-1]) optimizer_ins = self._acomplish_optimizer_ins(opt_ins, trainable) return loss, optimizer_ins, dict(), predictions, labels
def __init__( self, batch_size=32, num_layers=2, num_hidden_nodes=None, # a list which length is by 1 less than num_layers init_parameter=3., regularization_rate=.000006, additional_metrics=None, input_shape=None, num_classes=None, regime='autonomous_training', optimizer='adam', rho=0.95, # used for adadelta decay=0.9, # used for rmsprop ): if num_hidden_nodes is None: num_hidden_nodes = [1000, 1000] self._batch_size = batch_size self._num_layers = num_layers self._num_hidden_nodes = num_hidden_nodes self._init_parameter = init_parameter self._regularization_rate = regularization_rate self._input_shape = input_shape self._input_size = cumulative_mul(input_shape, 1) self._input_ndim = len(self._input_shape) self._num_classes = num_classes self._additional_metrics = additional_metrics self._optimizer = optimizer self._rho = rho self._decay = decay self._hooks = dict(inputs=None, labels=None, labels_prepared=None, train_op=None, learning_rate=None, momentum=None, loss=None, predictions=None, validation_inputs=None, validation_labels=None, validation_predictions=None, dropout=None, saver=None) for add_metric in self._additional_metrics: self._hooks[add_metric] = None self._hooks['validation_' + add_metric] = None self._dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob') self._hooks['dropout'] = self._dropout_keep_prob self._trainable = dict() self._inputs_and_labels_placeholders = dict() self._autonomous_train_specific_placeholders = dict() self._add_trainable_variables() self._add_inputs_and_labels() loss, optimizer_ins, _, predictions, labels = self.loss_and_opt_ins_for_inference( ) self._add_metrics_and_hooks(loss, predictions, labels) if regime == 'autonomous_training': self._add_autonomous_training_specific_placeholders() self._add_train_op(loss) elif regime == 'inference': pass elif regime == 'training_with_meta_optimizer': pass elif regime == 'optimizer_training': pass else: raise InvalidArgumentError( 'Not allowed regime', regime, 'regime', [ 'autonomous_training', 'inference', 'training_with_meta_optimizer', 'optimizer_training' ])
yscale = sys.argv[5] limit = sys.argv[6] if limit == 'None': limit = None else: limit = int(limit) shift = sys.argv[7] if limit == 'None': shift = None else: shift = int(shift) labels_and_files = sys.argv[8:] if len(labels_and_files) % 2 != 0: raise InvalidArgumentError( "Every file has to be provided with line name (number of arguments has to be even)", [labels_and_files, len(labels_and_files)], 'labels_and_files', "len(labels_and_files) % 2 == 0" ) line_names = [labels_and_files[2 * i] for i in range(len(labels_and_files) // 2)] file_names = [labels_and_files[2 * i + 1] for i in range(len(labels_and_files) // 2)] # print(line_names) data = list() for line_name, file_name in zip(line_names, file_names): with open(file_name, 'r') as f: lines = f.read().split('\n') lines = lines[:limit] while len(lines) > 0 and len(lines[-1]) == 0: lines = lines[:-1]