def test_is_generator(self): gen = TestIterator() self.assertTrue(is_generator(gen)) def generator(): yield ([1, 2], [1, 2]) self.assertTrue(is_generator(generator))
def test_is_generator(self): gen = TestIterator() self.assertTrue(is_generator(gen)) def generator(): yield ([1, 2], [1, 2]) self.assertTrue(is_generator(generator))
def job_start(job_backend, trainer, keras_callback): """ Starts the training of a job. Needs job_prepare() first. :type job_backend: JobBackend :type trainer: Trainer :return: """ job_backend.set_status('STARTING') job_model = job_backend.get_job_model() model_provider = job_model.get_model_provider() job_backend.set_status('LOAD DATA') datasets = job_model.get_datasets(trainer) print('trainer.input_shape = %s\n' % (simplejson.dumps(trainer.input_shape, default=invalid_json_values),)) print('trainer.classes = %s\n' % (simplejson.dumps(trainer.classes, default=invalid_json_values),)) multiple_inputs = len(datasets) > 1 insights_x = [] if multiple_inputs else None for dataset_name in job_model.get_input_dataset_names(): dataset = datasets[dataset_name] if is_generator(dataset['X_train']): batch_x, batch_y = dataset['X_train'].next() if multiple_inputs: insights_x.append(batch_x[0]) else: insights_x = batch_x[0] else: if multiple_inputs: insights_x.append(dataset['X_train'][0]) else: insights_x = dataset['X_train'][0] keras_callback.insights_x = insights_x print('Insights sample shape', keras_callback.insights_x.shape) keras_callback.write("Possible data keys '%s'\n" % "','".join(list(datasets.keys()))) data_train = model_provider.get_training_data(trainer, datasets) data_validation = model_provider.get_validation_data(trainer, datasets) keras_callback.set_validation_data(data_validation, trainer.nb_val_samples) trainer.set_status('CONSTRUCT') model = model_provider.get_model(trainer) trainer.set_model(model) trainer.set_status('COMPILING') loss = model_provider.get_loss(trainer) optimizer = model_provider.get_optimizer(trainer) model_provider.compile(trainer, model, loss, optimizer) model.summary() trainer.callbacks.append(keras_callback) model_provider.train(trainer, model, data_train, data_validation)
def set_validation_data(self, validation_data, validation_data_size=None): self.data_validation = validation_data self.data_validation_size = None if self.data_validation is None: return input_data_x = None # It's dict of AETROS code generation if isinstance(self.data_validation, dict) and 'x' in self.data_validation: if is_generator(self.data_validation['x']): # single input input_data_x = self.data_validation['x'] elif isinstance(self.data_validation['x'], dict): # multiple inputs named input_data_x = next(six.itervalues(self.data_validation['x'])) # Not from AETROS code generation else: if is_generator(self.data_validation): input_data_x = self.data_validation elif isinstance(self.data_validation, dict): input_data_x = next(six.itervalues(self.data_validation)) elif isinstance(self.data_validation, tuple): input_data_x = self.data_validation[0] if is_generator(input_data_x): if validation_data_size is None: raise Exception( 'validation_data_size needs to be set when a generator is given.' ) self.data_validation_size = validation_data_size elif input_data_x is not None: self.data_validation_size = len(input_data_x) if self.data_validation_size is None: raise Exception( 'data_validation_size could not be determined for given validation_data. Please specify it.' )
def set_validation_data(self, validation_data, validation_data_size=None): self.data_validation = validation_data self.data_validation_size = None if self.data_validation is None: return input_data_x = None # It's dict of AETROS code generation if isinstance(self.data_validation, dict) and 'x' in self.data_validation: if is_generator(self.data_validation['x']): # single input input_data_x = self.data_validation['x'] elif isinstance(self.data_validation['x'], dict): # multiple inputs named input_data_x = next(six.itervalues(self.data_validation['x'])) # Not from AETROS code generation else: if is_generator(self.data_validation): input_data_x = self.data_validation elif isinstance(self.data_validation, dict): input_data_x = next(six.itervalues(self.data_validation)) elif isinstance(self.data_validation, tuple): input_data_x = self.data_validation[0] if is_generator(input_data_x): if validation_data_size is None: raise Exception('validation_data_size needs to be set when a generator is given.') self.data_validation_size = validation_data_size elif input_data_x is not None: self.data_validation_size = len(input_data_x) if self.data_validation_size is None: raise Exception('data_validation_size could not be determined for given validation_data. Please specify it.')
def test_set_validation_data(self): job_backend = JobBackend('test') job_backend.job = {'id': 'test', 'index': 1, 'modelId': 'test/model'} keras_callback = KerasCallback(job_backend, sys.stdout) keras_callback.set_validation_data(([1, 2, 3], [1, 2, 3])) self.assertEqual(keras_callback.data_validation_size, 3) with pytest.raises(Exception): keras_callback.set_validation_data([[1, 2], [1, 2]]) # aetros format keras_callback.set_validation_data({'x': {'input': [1, 2, 3]}, 'y': {'output': [1, 2, 3]}}) self.assertEqual(keras_callback.data_validation_size, 3) keras_callback.set_validation_data({'input': [1, 2, 3], 'output': [1, 2, 3]}) self.assertEqual(keras_callback.data_validation_size, 3) keras_callback.set_validation_data(([1, 2, 3], [1, 2, 3]), 5) self.assertEqual(keras_callback.data_validation_size, 3) def generator(): yield ([1, 2], [1, 2]) self.assertTrue(is_generator(generator)) with pytest.raises(Exception): keras_callback.set_validation_data(generator) with pytest.raises(Exception): keras_callback.set_validation_data((generator, generator)) with pytest.raises(Exception): keras_callback.set_validation_data([generator, generator], 6) keras_callback.set_validation_data(generator, 5) self.assertEqual(keras_callback.data_validation_size, 5) keras_callback.set_validation_data((generator, generator), 4) self.assertEqual(keras_callback.data_validation_size, 4)
def build_confusion_matrix(self): confusion_matrix = {} if self.data_validation_size is None: return confusion_matrix if len(self.model.output_layers) > 1: return confusion_matrix first_input_layer = self.model.input_layers[0] first_output_layer = self.model.output_layers[0] if 'Softmax' not in str(first_output_layer.output) or len( first_output_layer.output_shape) != 2: return confusion_matrix input_data_x = None input_data_y = [] # It's dict of AETROS code generation if isinstance(self.data_validation, dict) and 'x' in self.data_validation: if is_generator(self.data_validation['x']): # single input input_data_x = self.data_validation['x'] elif isinstance(self.data_validation['x'], dict): # multiple inputs named input_data_x = self.data_validation['x'][ first_input_layer.name] input_data_y = self.data_validation['y'][ first_output_layer.name] # Not from AETROS code generation else: if is_generator(self.data_validation): input_data_x = self.data_validation elif isinstance(self.data_validation, dict): if len(self.model.input_layers) > 1: input_data_x = [] for layer in self.model.input_layers: input_data_x.append(self.data_validation[layer.name]) input_data_y = self.data_validation[first_output_layer.name] elif isinstance(self.data_validation, tuple): input_data_x = self.data_validation[0] input_data_y = self.data_validation[1] if input_data_x is None: return confusion_matrix matrix = np.zeros((first_output_layer.output_shape[1], first_output_layer.output_shape[1])) if is_generator(input_data_x): processed_samples = 0 while processed_samples < self.data_validation_size: generator_output = next(input_data_x) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: self.model._stop.set() raise Exception('output of generator should be a tuple ' '(x, y, sample_weight) ' 'or (x, y). Found: ' + str(generator_output)) if type(x) is list: nb_samples = len(x[0]) elif type(x) is dict: nb_samples = len(list(x.values())[0]) else: nb_samples = len(x) processed_samples += nb_samples prediction = self.model.predict_on_batch(x) predicted_classes = prediction.argmax(axis=-1) expected_classes = y.argmax(axis=-1) try: for sample_idx, predicted_class in enumerate( predicted_classes): expected_class = expected_classes[sample_idx] matrix[expected_class, predicted_class] += 1 except: pass else: batch_size = self.current[ 'batch_size'] if 'batch_size' in self.current else 16 prediction = self.model.predict(input_data_x, batch_size=batch_size) predicted_classes = prediction.argmax(axis=-1) expected_classes = np.array(input_data_y).argmax(axis=-1) try: for sample_idx, predicted_class in enumerate( predicted_classes): expected_class = expected_classes[sample_idx] matrix[expected_class, predicted_class] += 1 except: pass confusion_matrix[first_output_layer.name] = matrix.tolist() return confusion_matrix
def job_start(job_backend, trainer, keras_callback): """ Starts the training of a job. Needs job_prepare() first. :type job_backend: JobBackend :type trainer: Trainer :return: """ job_backend.set_status('STARTING') job_model = job_backend.get_job_model() model_provider = job_model.get_model_provider() job_backend.set_status('LOAD DATA') datasets = job_model.get_datasets(trainer) print( 'trainer.input_shape = %s\n' % (simplejson.dumps(trainer.input_shape, default=invalid_json_values), )) print('trainer.classes = %s\n' % (simplejson.dumps(trainer.classes, default=invalid_json_values), )) multiple_inputs = len(datasets) > 1 insights_x = [] if multiple_inputs else None for dataset_name in job_model.get_input_dataset_names(): dataset = datasets[dataset_name] if is_generator(dataset['X_train']): batch_x, batch_y = dataset['X_train'].next() if multiple_inputs: insights_x.append(batch_x[0]) else: insights_x = batch_x[0] else: if multiple_inputs: insights_x.append(dataset['X_train'][0]) else: insights_x = dataset['X_train'][0] keras_callback.insights_x = insights_x print('Insights sample shape', keras_callback.insights_x.shape) keras_callback.write("Possible data keys '%s'\n" % "','".join(list(datasets.keys()))) data_train = model_provider.get_training_data(trainer, datasets) data_validation = model_provider.get_validation_data(trainer, datasets) keras_callback.set_validation_data(data_validation, trainer.nb_val_samples) trainer.set_status('CONSTRUCT') model = model_provider.get_model(trainer) trainer.set_model(model) trainer.set_status('COMPILING') loss = model_provider.get_loss(trainer) optimizer = model_provider.get_optimizer(trainer) model_provider.compile(trainer, model, loss, optimizer) model.summary() trainer.callbacks.append(keras_callback) model_provider.train(trainer, model, data_train, data_validation)
def build_confusion_matrix(self): confusion_matrix = {} if self.data_validation_size is None: return confusion_matrix if len(self.model.output_layers) > 1: return confusion_matrix first_input_layer = self.model.input_layers[0] first_output_layer = self.model.output_layers[0] if 'Softmax' not in str(first_output_layer.output) or len(first_output_layer.output_shape) != 2: return confusion_matrix input_data_x = None input_data_y = [] # It's dict of AETROS code generation if isinstance(self.data_validation, dict) and 'x' in self.data_validation: if is_generator(self.data_validation['x']): # single input input_data_x = self.data_validation['x'] elif isinstance(self.data_validation['x'], dict): # multiple inputs named input_data_x = self.data_validation['x'][first_input_layer.name] input_data_y = self.data_validation['y'][first_output_layer.name] # Not from AETROS code generation else: if is_generator(self.data_validation): input_data_x = self.data_validation elif isinstance(self.data_validation, dict): if len(self.model.input_layers) > 1: input_data_x = [] for layer in self.model.input_layers: input_data_x.append(self.data_validation[layer.name]) input_data_y = self.data_validation[first_output_layer.name] elif isinstance(self.data_validation, tuple): input_data_x = self.data_validation[0] input_data_y = self.data_validation[1] if input_data_x is None: return confusion_matrix matrix = np.zeros((first_output_layer.output_shape[1], first_output_layer.output_shape[1])) if is_generator(input_data_x): processed_samples = 0 while processed_samples < self.data_validation_size: generator_output = next(input_data_x) if len(generator_output) == 2: x, y = generator_output sample_weight = None elif len(generator_output) == 3: x, y, sample_weight = generator_output else: self.model._stop.set() raise Exception('output of generator should be a tuple ' '(x, y, sample_weight) ' 'or (x, y). Found: ' + str(generator_output)) if type(x) is list: nb_samples = len(x[0]) elif type(x) is dict: nb_samples = len(list(x.values())[0]) else: nb_samples = len(x) processed_samples += nb_samples prediction = self.model.predict_on_batch(x) predicted_classes = prediction.argmax(axis=-1) expected_classes = y.argmax(axis=-1) try: for sample_idx, predicted_class in enumerate(predicted_classes): expected_class = expected_classes[sample_idx] matrix[expected_class, predicted_class] += 1 except Exception: pass else: batch_size = self.current['batch_size'] if 'batch_size' in self.current else 16 prediction = self.model.predict(input_data_x, batch_size=batch_size) predicted_classes = prediction.argmax(axis=-1) expected_classes = np.array(input_data_y).argmax(axis=-1) try: for sample_idx, predicted_class in enumerate(predicted_classes): expected_class = expected_classes[sample_idx] matrix[expected_class, predicted_class] += 1 except Exception: pass confusion_matrix[first_output_layer.name] = matrix.tolist() return confusion_matrix