def __init__(self, keras_model, loss, worker_optimizer): self.master_model = serialize_keras_model(keras_model) self.loss = loss self.worker_optimizer = worker_optimizer self.history = [] self.training_time_start = 0 self.training_time_end = 0 self.training_time = 0
def __init__(self, keras_model, loss, worker_optimizer): set_keras_base_directory() self.master_model = serialize_keras_model(keras_model) self.loss = loss self.worker_optimizer = worker_optimizer self.history = [] self.training_time_start = 0 self.training_time_end = 0 self.training_time = 0 self.max_mini_batches_prefetch = 100
def __init__(self, keras_model, loss, worker_optimizer, metrics=["accuracy"], loss_weights=None): set_keras_base_directory() self.master_model = serialize_keras_model(keras_model) self.loss = loss self.loss_weights = loss_weights self.worker_optimizer = worker_optimizer self.metrics = metrics self.history = [] self.training_time_start = 0 self.training_time_end = 0 self.training_time = 0 self.max_mini_batches_prefetch = 100
def destroy_job(): secret = request.args.get('secret') job = self.get_submitted_job(secret) if job is not None and not job.running(): with self.mutex: model = self.jobs[secret].get_trained_model() history = self.jobs[secret].get_history() model = pickle_object( serialize_keras_model(model)).encode('hex_codec') history = pickle_object(history).encode('hex_codec') d = {} d['model'] = model d['history'] = history del self.jobs[secret] return json.dumps(d), 200 return '', 400
def average_models(self, models): """Averages the specified list of Keras models, and assigns the averaged model as the master model. # Arguments: models: list. A list of serialized Keras models. """ num_models = len(models) # Get all weights of the models. for i in range(0, num_models): weights = np.asarray(deserialize_keras_model(models[i]).get_weights()) self.parameter_buffer += weights # Average the parameters. self.parameter_buffer /= num_models temp_model = deserialize_keras_model(self.master_model) temp_model.set_weights(self.parameter_buffer) self.master_model = serialize_keras_model(temp_model)
def train(self, worker_id, iterator): """Training procedure for the worker node. # Arguments worker_id: int. Partition index provided by Spark. Can be used as a worker_id. iterator: iterator. Data iterator. """ # Prepare the optimization procedure. self.start_prefetching_thread(iterator) self.set_worker_id(worker_id) self.prepare_model() # Start the optimization procedure. try: self.optimize() except: pass # Wait for the prefetching thread to stop. self.prefetching_thread.join() return iter([serialize_keras_model(self.model)])
def train(self, worker_id, iterator): """Training procedure for the worker node. # Arguments worker_id: int. Partition index provided by Spark. Can be used as a worker_id. iterator: iterator. Data iterator. """ # Prepare the optimization procedure. self.start_prefetching_thread(iterator) self.set_worker_id(worker_id) self.prepare_model() # Start the optimization procedure. try: self.optimize() except Exception as e: # Stop the prefetching process. self.is_prefetching = False print(e) # Wait for the prefetching thread to stop. self.prefetching_thread.join() return iter([serialize_keras_model(self.model)])
def train(self, worker_id, iterator): """Training procedure with sequential gradient updates. # Returns Trained serialized Keras model. """ # Prepare the model. self.prepare_model() try: while True: # Fetch the next mini-batch. batch = [next(iterator) for _ in range(self.batch_size)] # Retrieve the feature and label vectors. feature_iterator, label_iterator = tee(batch, 2) X = np.asarray( [x[self.features_column] for x in feature_iterator]) Y = np.asarray([x[self.label_column] for x in label_iterator]) self.model.train_on_batch(X, Y) except StopIteration: pass return iter([serialize_keras_model(self.model)])
def __init__(self, keras_model): self.model = serialize_keras_model(keras_model)
def set_model(self, model): """Sets the master model to be used by the trainer.""" self.master_model = serialize_keras_model(model)
def __init__(self, keras_model): self.model = serialize_keras_model(keras_model)
def set_model(self, model): """Sets the master model to be used by the trainer.""" self.master_model = serialize_keras_model(model)