def save(self): dependency = self.gather_dependency() export_data = { "version": "1.0", "dependency": dependency, "processor": self.processor_builder.serialize( create_dir_if_needed(self.export_dir / "asset" / "processor")), "model": self.model_builder.serialize( create_dir_if_needed(self.export_dir / "asset" / "model")), "metadata": self.metadata_builder.serialize( create_dir_if_needed(self.export_dir / "asset" / "metadata")), } if self.remote_model_builder: export_data["remote_model"] = self.remote_model_builder.serialize( create_dir_if_needed(self.export_dir / "asset" / "remote_model")) metadata_file = self.export_dir / "metadata.json" with metadata_file.open("wt") as fd: json.dump(export_data, fd) return export_data
def main(): raw_config = read_configure() model = Model(raw_config) config = model.get_default_config() config.update(raw_config) corpus = get_corpus_processor(config) corpus.prepare() train_data_generator_func = corpus.get_generator_func(corpus.TRAIN) eval_data_generator_func = corpus.get_generator_func(corpus.EVAL) corpus_meta_data = corpus.get_meta_info() config["tags_data"] = generate_tagset(corpus_meta_data["tags"]) # train and evaluate model train_input_func = build_input_func(train_data_generator_func, config) eval_input_func = (build_input_func(eval_data_generator_func, config) if eval_data_generator_func else None) evaluate_result, export_results, final_saved_model = model.train_and_eval_then_save( train_input_func, eval_input_func, config) export_as_deliverable_model(create_dir_if_needed( config["deliverable_model_dir"]), tensorflow_saved_model=final_saved_model, converter_for_request=converter_for_request, converter_for_response=converter_for_response, addition_model_dependency=["micro_toolkit"])
def main(): raw_config = read_configure() model = Model(raw_config) config = model.get_default_config() # 默认的配置 config.update(raw_config) # mlflow hyperparameter mlflow.log_param("Batch_Size", config["batch_size"]) mlflow.log_param("Learning_Rate", config["learning_rate"]) mlflow.log_param("Epochs", config["epochs"]) mlflow.log_param("Embedding_Dim", config["embedding_dim"]) corpus = get_corpus_processor(config) corpus.prepare() train_data_generator_func = corpus.get_generator_func(corpus.TRAIN) eval_data_generator_func = corpus.get_generator_func(corpus.EVAL) corpus_meta_data = corpus.get_meta_info() config["tags_data"] = generate_tagset(corpus_meta_data["tags"]) # train and evaluate model train_input_func = build_input_func(train_data_generator_func, config) eval_input_func = (build_input_func(eval_data_generator_func, config) if eval_data_generator_func else None) evaluate_result, export_results, final_saved_model = model.train_and_eval_then_save( train_input_func, eval_input_func, config) # mlflow metrics # mlflow.log_metrics(evaluate_result, step=100) # mlflow Logging the saved model mlflow.tensorflow.log_model(tf_saved_model_dir=final_saved_model, tf_meta_graph_tags=[tag_constants.SERVING], tf_signature_def_key='serving_default', artifact_path='model') export_as_deliverable_model(create_dir_if_needed( config["deliverable_model_dir"]), tensorflow_saved_model=final_saved_model, converter_for_request=ConverterForRequest(), converter_for_response=ConverterForResponse(), addition_model_dependency=["micro_toolkit"])
def serialize(self, asset_dir: Path): instance = {} for ( processor_instance_name, processor_instance, ) in self.processor_instance_registry.items(): processor_instance_asset_dir = create_dir_if_needed( asset_dir / processor_instance_name) processor_instance.serialize(processor_instance_asset_dir) instance[processor_instance_name] = { "class": get_class_fqn_name(processor_instance), "parameter": processor_instance.get_config(), } pipeline = { "pre": self.preprocess_pipeline, "post": self.postprocess_pipeline } return {"version": "1.0", "instance": instance, "pipeline": pipeline}