def load_train(self, config: BuildModelConfig, model_config_path: str=None, model_weights_path: str=None): with open(model_config_path, "r", encoding='utf-8') as f: if config.use_crf: from tensorflow.keras_contrib.layers import CRF custom_objects = {'ReversedLSTM': ReversedLSTM, 'CRF': CRF} self.train_model = model_from_yaml(f.read(), custom_objects=custom_objects) else: custom_objects = {'ReversedLSTM': ReversedLSTM} self.train_model = model_from_yaml(f.read(), custom_objects=custom_objects) self.train_model.load_weights(model_weights_path) loss = {} metrics = {} if config.use_crf: out_layer_name = 'crf' offset = 0 if config.use_pos_lm: offset += 2 if config.use_word_lm: offset += 2 loss[out_layer_name] = self.train_model.layers[-1-offset].loss_function metrics[out_layer_name] = self.train_model.layers[-1-offset].accuracy else: out_layer_name = 'main_pred' loss[out_layer_name] = 'sparse_categorical_crossentropy' metrics[out_layer_name] = 'accuracy' if config.use_pos_lm: prev_layer_name = 'shifted_pred_prev' next_layer_name = 'shifted_pred_next' loss[prev_layer_name] = loss[next_layer_name] = 'sparse_categorical_crossentropy' metrics[prev_layer_name] = metrics[next_layer_name] = 'accuracy' self.train_model.compile(Adam(clipnorm=5.), loss=loss, metrics=metrics) self.eval_model = Model(inputs=self.train_model.inputs, outputs=self.train_model.outputs[0])
def load_eval(self, config: BuildModelConfig, eval_model_config_path: str, eval_model_weights_path: str) -> None: with open(eval_model_config_path, "r", encoding='utf-8') as f: if config.use_crf: from tensorflow.keras_contrib.layers import CRF custom_objects = {'ReversedLSTM': ReversedLSTM, 'CRF': CRF} self.eval_model = model_from_yaml(f.read(), custom_objects=custom_objects) else: custom_objects = {'ReversedLSTM': ReversedLSTM} self.eval_model = model_from_yaml(f.read(), custom_objects=custom_objects) self.eval_model.load_weights(eval_model_weights_path) self.eval_model._make_predict_function()
def load_keras_model(arch_fpath, weights_fpath, custom_objects=None, n_gpus=0): """Load a Keras architecture and weights Parameters ---------- arch_fpath: str Architecture saved as YAML file weights_fpath: str Weights saved as h5 file custom_objects: dicts Custom objects needed when loading the Keras model. n_gpus: int Number of gpus available to run prediction. Default 0. Returns ------- model: keras.engine.training.Model """ if not op.splitext(arch_fpath)[-1] == '.yaml': raise ValueError('Model filepath must have `.yaml` extension.') if not op.splitext(weights_fpath)[-1] == '.h5': raise ValueError('Weights filepath must have `.h5` extension.') with open(arch_fpath, "r") as yaml_file: yaml_architecture = yaml_file.read() if n_gpus > 1: # Load weights on CPU to avoid taking up GPU space with tf.device('/cpu:0'): template_model = model_from_yaml(yaml_architecture, custom_objects=custom_objects) template_model.load_weights(weights_fpath) for layer in template_model.layers: layer.trainable = False model = multi_gpu_model(template_model, gpus=n_gpus) # If on only 1 gpu (or cpu), train as normal else: model = model_from_yaml(yaml_architecture, custom_objects=custom_objects) model.load_weights(weights_fpath) for layer in model.layers: layer.trainable = False return model
def predict(): yaml_file = open("../sensing_data/models/dnn_tf_1_1.yaml", 'r') loaded_model_yaml = yaml_file.read() yaml_file.close() dnn_pred = model_from_yaml(loaded_model_yaml) # load weights into new model dnn_pred.load_weights("../sensing_data/models/dnn_tf_1_1.h5") print("Loaded model from disk") dnn_pred.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy']) dnn_pred.summary() X, y, shape = data.load_prediction(ratio=1, normalize=False, osm_roads=False, split_struct=False, army_gt=False) normalizer = preprocessing.Normalizer().fit(X) X = normalizer.transform(X) y_pred = dnn_pred.predict(X) y_pred = [np.argmax(pred) for pred in tqdm(y_pred)] kappa = cohen_kappa_score(y - 1, y_pred) print(f'Kappa: {kappa}') print(classification_report(y - 1, y_pred)) y_pred = np.array(y_pred) yr = y_pred.reshape(shape) viz.createGeotiff(OUT_RASTER, yr, REF_FILE, gdal.GDT_Byte)
def load(self, model_config_path: str, model_weights_path: str) -> None: with open(model_config_path, "r", encoding='utf-8') as f: self.model = model_from_yaml(f.read()) self.model.load_weights(model_weights_path) self.char_layer = TimeDistributed( Model(self.model.input_layers[0].output, self.model.layers[-2].input))
def _fit(self, df): """Private fit method of the Estimator, which trains the model. """ simple_rdd = df_to_simple_rdd( df, categorical=self.get_categorical_labels(), nb_classes=self.get_nb_classes(), features_col=self.getFeaturesCol(), label_col=self.getLabelCol()) simple_rdd = simple_rdd.repartition(self.get_num_workers()) keras_model = model_from_yaml(self.get_keras_model_config()) metrics = self.get_metrics() loss = self.get_loss() optimizer = get_optimizer(self.get_optimizer_config()) keras_model.compile(loss=loss, optimizer=optimizer, metrics=metrics) spark_model = SparkModel(model=keras_model, mode=self.get_mode(), frequency=self.get_frequency(), num_workers=self.get_num_workers()) spark_model.fit(simple_rdd, epochs=self.get_epochs(), batch_size=self.get_batch_size(), verbose=self.get_verbosity(), validation_split=self.get_validation_split()) model_weights = spark_model.master_network.get_weights() weights = simple_rdd.ctx.broadcast(model_weights) return ElephasTransformer( labelCol=self.getLabelCol(), outputCol='prediction', keras_model_config=spark_model.master_network.to_yaml(), weights=weights, loss=loss)
def train(self, data_iterator): """Train a keras model on a worker """ history = None optimizer = get_optimizer(self.master_optimizer) self.model = model_from_yaml(self.yaml, self.custom_objects) self.model.compile(optimizer=optimizer, loss=self.master_loss, metrics=self.master_metrics) self.model.set_weights(self.parameters.value) feature_iterator, label_iterator = tee(data_iterator, 2) x_train = np.asarray([x for x, y in feature_iterator]) y_train = np.asarray([y for x, y in label_iterator]) weights_before_training = self.model.get_weights() if x_train.shape[0] > self.train_config.get('batch_size'): history = self.model.fit(x_train, y_train, **self.train_config) weights_after_training = self.model.get_weights() deltas = subtract_params(weights_before_training, weights_after_training) if history: yield [deltas, history.history] else: yield [deltas, None]
def __init__(self, corpus_path, model_path, weights_path): with open(corpus_path, encoding='utf-16') as corpus_file: self.corpus = corpus_file.read() # Get a unique identifier for each char in the corpus, # then make some dicts to ease encoding and decoding self.chars = sorted(list(set(self.corpus))) self.encoding = {c: i for i, c in enumerate(self.chars)} self.decoding = {i: c for i, c in enumerate(self.chars)} print(len(self.chars)) print(self.chars) # Some fields we'll need later self.num_chars = len(self.chars) self.sentence_length = 50 self.corpus_length = len(self.corpus) # Build our network from loaded architecture and weights with open(model_path) as model_file: architecture = model_file.read() self.model = model_from_yaml(architecture) self.model.load_weights(weights_path) self.model.compile(loss='categorical_crossentropy', optimizer='adam')
def load(self): log.info("loading project %s ..." % self.path) if not self.exists(): return "%s does not exist" % self.path err = self.logic.load() if err is not None: return err if os.path.exists(self.weights_path): log.debug("loading model from %s ...", self.weights_path) self.model = load_model(self.weights_path) # https://github.com/keras-team/keras/issues/6462 self.model.make_predict_function() elif os.path.exists(self.model_path): log.debug("loading model from %s ...", self.model_path) with open(self.model_path, 'r') as fp: self.model = model_from_yaml(fp.read()) if os.path.exists(self.history_path): log.debug("loading history from %s ...", self.history_path) with open(self.history_path, 'r') as fp: self.history = json.load(fp) if os.path.exists(self.classes_path): log.debug("loading classes from %s ...", self.classes_path) with open(self.classes_path, 'r') as fp: self.classes = {int(k): v for k, v in json.load(fp).items()} return None
def load_model(self): print("Load model...") # open model file. model_path = self.yml["Modelsetting"]["model_path"] if "json" in model_path: model = model_from_json(open(model_path).read()) elif "yaml" in model_path: model = model_from_yaml(open(model_path).read()) else: model = load_model(model_path) # Read the weight of the model. if self.yml["Modelsetting"]["retrain_model"]: print("Load model weight.") model.load_weights(self.yml["Modelsetting"]["weight_path"]) # model freeze? if not self.yml["Modelsetting"]["trainable"]: model.trainable = False # model compile. summury. model.compile(loss=self.set_modelloss(self.yml["Modelsetting"]["model_loss"]), optimizer=self.set_optimizers(self.yml["Modelsetting"]["optimizers"], self.yml["Trainsetting"]["learnrate"]), metrics=["accuracy"]) print("compile ok. summmury") # Display the results of the compiled model. model.summary() return model
def extract_features_and_predict(model_yaml: str, custom_objects: dict, features_col: str, model_type: ModelType, data): model = model_from_yaml(model_yaml, custom_objects) model.set_weights(weights.value) predict_function = determine_predict_function(model, model_type) return predict_function( np.stack([from_vector(x[features_col]) for x in data]))
def load_model_from_yaml(self, file_name): yaml_model = open(file_name, 'r') loaded_model_yaml = yaml_model.read() yaml_model.close() loaded_model = model_from_yaml(loaded_model_yaml) return loaded_model
def _evaluate(model, optimizer, loss, custom_objects, metrics, kwargs, data_iterator): model = model_from_yaml(model, custom_objects) model.compile(optimizer, loss, metrics) model.set_weights(weights.value) feature_iterator, label_iterator = tee(data_iterator, 2) x_test = np.asarray([x for x, y in feature_iterator]) y_test = np.asarray([y for x, y in label_iterator]) return [model.evaluate(x_test, y_test, **kwargs)]
def loadKerasModel(filepath, string_model=None, format_export='json'): if string_model is None: with open(filepath, 'r') as file: string_model = file.read() if format_export == 'json': model = model_from_json(string_model) else: model = model_from_yaml(string_model) return model
def loadModel(yamlPathName, h5PathName): with open(yamlPathName + '.yaml', 'r') as yaml_file: loaded_model_yaml = yaml_file.read() loaded_model = model_from_yaml(loaded_model_yaml) loaded_model.load_weights(h5PathName + '.h5') loaded_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) return loaded_model
def train(self, data_iterator): """Train a keras model on a worker and send asynchronous updates to parameter server """ feature_iterator, label_iterator = tee(data_iterator, 2) x_train = np.asarray([x for x, y in feature_iterator]) y_train = np.asarray([y for x, y in label_iterator]) if x_train.size == 0: return self.model = model_from_yaml(self.yaml, self.custom_objects) self.model.compile(optimizer=get_optimizer(self.master_optimizer), loss=self.master_loss, metrics=self.master_metrics) self.model.set_weights(self.parameters.value) epochs = self.train_config['epochs'] batch_size = self.train_config.get('batch_size') nb_train_sample = x_train.shape[0] nb_batch = int(np.ceil(nb_train_sample / float(batch_size))) index_array = np.arange(nb_train_sample) batches = [(i * batch_size, min(nb_train_sample, (i + 1) * batch_size)) for i in range(0, nb_batch)] if self.frequency == 'epoch': for epoch in range(epochs): weights_before_training = self.client.get_parameters() self.model.set_weights(weights_before_training) self.train_config['epochs'] = 1 if x_train.shape[0] > batch_size: self.model.fit(x_train, y_train, **self.train_config) self.train_config['epochs'] = epochs weights_after_training = self.model.get_weights() deltas = subtract_params(weights_before_training, weights_after_training) self.client.update_parameters(deltas) elif self.frequency == 'batch': for epoch in range(epochs): if x_train.shape[0] > batch_size: for (batch_start, batch_end) in batches: weights_before_training = self.client.get_parameters() self.model.set_weights(weights_before_training) batch_ids = index_array[batch_start:batch_end] x = slice_arrays(x_train, batch_ids) y = slice_arrays(y_train, batch_ids) self.model.train_on_batch(x, y) weights_after_training = self.model.get_weights() deltas = subtract_params(weights_before_training, weights_after_training) self.client.update_parameters(deltas) else: raise ValueError( 'frequency parameter can be `epoch` or `batch, got {}'.format( self.frequency)) yield []
def loadModel(MODEL_NAME): print('Loading Model..') model = model_from_yaml(open(MODEL_NAME + '.yaml').read()) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.load_weights(MODEL_NAME + '.h5') model.summary() print('Done') return model
def load_model(): y_fil = os.path.join(MODEL_FOLDER, GENERATOR_YAML) w_fil = os.path.join(MODEL_FOLDER, GENERATOR_H5) with open(y_fil, 'r') as yf: yaml = yf.read() model = model_from_yaml(yaml) model.load_weights(w_fil) return model
def load(path): output_dir = Path(path) with open(output_dir / 'model_config.yaml') as file: model_config = file.read() model = model_from_yaml(model_config) model.load_weights(output_dir / "model.h5") return model
def load_model(input_model_path, input_json_path=None, input_yaml_path=None): if not Path(input_model_path).exists(): raise FileNotFoundError('Model file `{}` does not exist.'.format(input_model_path)) try: # model = keras.models.load_model(input_model_path, compile=False) with open(input_json_path) as json_file: json_config = json_file.read() model = tf.keras.models.model_from_json(json_config, custom_objects={'tf': tf}) # Load weights model.load_weights(input_model_path) return model except FileNotFoundError as err: logging.error('Input mode file (%s) does not exist.', FLAGS.input_model) raise err except ValueError as wrong_file_err: if input_json_path: if not Path(input_json_path).exists(): raise FileNotFoundError( 'Model description json file `{}` does not exist.'.format( input_json_path)) try: model = model_from_json(open(str(input_json_path)).read()) model.load_weights(input_model_path) return model except Exception as err: logging.error("Couldn't load model from json.") raise err elif input_yaml_path: if not Path(input_yaml_path).exists(): raise FileNotFoundError( 'Model description yaml file `{}` does not exist.'.format( input_yaml_path)) try: model = model_from_yaml(open(str(input_yaml_path)).read()) model.load_weights(input_model_path) return model except Exception as err: logging.error("Couldn't load model from yaml.") raise err else: logging.error( 'Input file specified only holds the weights, and not ' 'the model definition. Save the model using ' 'model.save(filename.h5) which will contain the network ' 'architecture as well as its weights. ' 'If the model is saved using the ' 'model.save_weights(filename) function, either ' 'input_model_json or input_model_yaml flags should be set to ' 'to import the network architecture prior to loading the ' 'weights. \n' 'Check the keras documentation for more details ' '(https://keras.io/getting-started/faq/)') raise wrong_file_err
def load_model(path="results/"): yaml_file = open(path + 'model.yaml', 'r') loaded_model_yaml = yaml_file.read() yaml_file.close() loaded_model = models.model_from_yaml(loaded_model_yaml) # load weights into new model loaded_model.load_weights( "machine_learning/models/pl_sb_model/model_saved/model.h5") print("Loaded model from disk") return loaded_model
def get_model(): print('loading model......') #从yaml中恢复LSTM模型配置 with open('../model/lstm.yml', 'r') as f: yaml_string = yaml.load(f) model = model_from_yaml(yaml_string) #model = build_model() model.load_weights('../model/lstm.h5') model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def load_from_yaml_with_weights(file_path, weight_path=None, custom_objects=None, **kwargs): ''' ''' if weight_path is None: file_path, weight_path = find_architecture_and_weight_paths(file_path) with open(file_path, 'r') as fin: model = model_from_yaml(fin.read(), custom_objects=custom_objects) model.load_weights(weight_path) return model
def load_model(model_fpath, weights_fpath): """Load a model from yaml architecture and h5 weights.""" assert model_fpath[-5:] == '.yaml' assert weights_fpath[-3:] == '.h5' with open(model_fpath, "r") as yaml_file: yaml_architecture = yaml_file.read() model = model_from_yaml(yaml_architecture) model.load_weights(weights_fpath) return model
def extract_features_and_predict(model_yaml: str, custom_objects: dict, features_col: str, data, inference_batch_size: int = None): model = model_from_yaml(model_yaml, custom_objects) model.set_weights(weights.value) if inference_batch_size is not None and inference_batch_size > 0: return batched_prediction(data, inference_batch_size, features_col, model.predict) else: return model.predict( np.array([from_vector(x[features_col]) for x in data]))
def load_model(model_fpath, weights_fpath): """Load a model from yaml architecture and h5 weights.""" assert model_fpath[-5:] == '.yaml' assert weights_fpath[-3:] == '.h5' with open(model_fpath, "r") as yaml_file: yaml_architecture = yaml_file.read() model = model_from_yaml(yaml_architecture) # Issue when using pyyaml 5.3.1, downgrading to 3.13 model.load_weights(weights_fpath) return model
def load_data(): d = load("labelencoder1_X.joblib1.dat") yaml_file = open('model.yaml', 'r') loaded_model_yaml1 = yaml_file.read() yaml_file.close() loaded_model2 = model_from_yaml(loaded_model_yaml1) # load weights into new model loaded_model2.load_weights("model.h5") print("Loaded model from disk") test_case = load("test_case1.joblib1.dat") test_case = array(test_case).reshape(1, 1, 6) return loaded_model2, test_case
def reload_model(self): K.clear_session() if os.path.exists(self.weights_path): self.model = load_model(self.weights_path) # https://github.com/keras-team/keras/issues/6462 self.model._make_predict_function() elif os.path.exists(self.model_path): with open(self.model_path, 'r') as fp: self.model = model_from_yaml(fp.read()) else: self.model = self.logic.builder(True) gc.collect()
def loadModels(cls, filename, type): # load json and create model if type == 'json': json_file = open(filename, 'r') loaded_model_json = json_file.read() json_file.close() loaded_model = model_from_json(loaded_model_json) elif type == 'yaml': yaml_file = open(filename, 'r') loaded_model_yaml = yaml_file.read() yaml_file.close() loaded_model = model_from_yaml(loaded_model_yaml) return loaded_model
def lstm_predict(string): print('loading model......') with open('./store/lstm.yml', 'r') as f: yaml_string = yaml.load(f) model = model_from_yaml(yaml_string) print('loading weights......') model.load_weights('./store/lstm.h5') model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) data = input_transform(string) data.reshape(1, -1) result = model.predict_classes(data) return result