def keras_train_and_save(estimator, model_params, save, FLAGS, train_dataset_fn, val_dataset_fn, label_meta, epochs, verbose, metric_names, validation_steps, load_pretrained_model, model_meta): print("Start training using keras model...") classifier, has_none_optimizer = keras_compile(estimator, model_params, save, metric_names) train_dataset = train_dataset_fn() if val_dataset_fn is not None: validate_dataset = val_dataset_fn() else: validate_dataset = None if load_pretrained_model: # FIXME(typhoonzero): copied from runtime.tensorflow.train_keras inputs, targets = next(iter(train_dataset.take(1))) classifier.evaluate(inputs, targets) classifier.load_weights(save) if len(FLAGS.worker_hosts.split(",")) > 1: keras_train_distributed(classifier, model_params, save, model_meta, FLAGS, train_dataset_fn, val_dataset_fn) else: keras_train_compiled(classifier, save, train_dataset, validate_dataset, label_meta, epochs, verbose, model_meta, validation_steps, has_none_optimizer) print("saving keras model to: %s" % FLAGS.sqlflow_oss_modeldir) if len(FLAGS.worker_hosts.split(",")) > 1: oss.save_dir(FLAGS.sqlflow_oss_modeldir, save) else: oss.save_file(FLAGS.sqlflow_oss_modeldir, save) oss.save_file(FLAGS.sqlflow_oss_modeldir, "model_meta.json")
def save_to_oss(self, oss_model_dir, local_dir=None): """ This save function would archive all the files on local_dir into a tarball, and save it into OSS model directory. Args: oss_model_dir (str): the OSS model directory to save. It is in the format of oss://bucket/path/to/dir/. local_dir (str): the local directory to save. Returns: None. """ if local_dir is None: local_dir = os.getcwd() with temp_file.TemporaryDirectory() as tmp_dir: tarball = os.path.join(tmp_dir, TARBALL_NAME) self._zip(local_dir, tarball) oss.save_file(oss_model_dir, tarball, TARBALL_NAME) with temp_file.TemporaryDirectory() as tmp_dir: model_obj_file = os.path.join(tmp_dir, MODEL_OBJ_FILE_NAME) with open(model_obj_file, "w") as f: f.write( json.dumps(self._to_dict(), cls=JSONEncoderWithFeatureColumn)) oss.save_file(oss_model_dir, model_obj_file, MODEL_OBJ_FILE_NAME)
def save_model(model_dir, filename, model_params, train_params, feature_metas, feature_column_names, label_meta, fc_map_ir): oss.save_file(model_dir, filename) oss.save_file(model_dir, "{}.pmml".format(filename)) oss.save_metas( model_dir, 1, "xgboost_model_desc", "", # estimator = "" model_params, train_params, feature_metas, feature_column_names, label_meta, fc_map_ir)
def keras_train_and_save_legacy(estimator, model_params, save, FLAGS, train_dataset_fn, val_dataset_fn, label_meta, epochs, verbose, metric_names, validation_steps, load_pretrained_model, model_meta, is_pai): print("Start training using keras model...") try: classifier, has_none_optimizer = keras_compile(estimator, model_params, metric_names) except Exception as e: if hasattr(estimator, "sqlflow_train_loop"): sys.stderr.write( "compile keras model failed, ignoring this error " "since the model seems to defined sqlflow_train_loop.") classifier = init_model_with_feature_column( estimator, model_params, has_none_optimizer=True) has_none_optimizer = True else: raise e train_dataset = train_dataset_fn() if val_dataset_fn is not None: validate_dataset = val_dataset_fn() else: validate_dataset = None if load_pretrained_model: # Must run one batch to initialize parameters before load_weights inputs, targets = next(iter(train_dataset.take(1))) classifier.evaluate(inputs, targets) # NOTE(sneaxiy): should we save/load optimizer info for incremental # training, or let users to write the same WITH statements in SQL? load_keras_model_weights(classifier, save) if len(FLAGS.worker_hosts.split(",")) > 1: keras_train_distributed(classifier, model_params, save, model_meta, FLAGS, train_dataset_fn, val_dataset_fn, is_pai) else: keras_train_compiled(classifier, save, train_dataset, validate_dataset, label_meta, epochs, verbose, model_meta, validation_steps, has_none_optimizer) if is_pai: print("saving keras model to: %s" % FLAGS.sqlflow_oss_modeldir) oss.save_dir(FLAGS.sqlflow_oss_modeldir, save) oss.save_file(FLAGS.sqlflow_oss_modeldir, "model_meta.json")
def keras_train_and_save(estimator, model_params, save, FLAGS, train_dataset_fn, val_dataset_fn, label_meta, epochs, verbose, metric_names, validation_steps, load, model_meta, is_pai): print("Start training using keras model...") try: classifier, has_none_optimizer = keras_compile(estimator, model_params, metric_names) except Exception: if hasattr(estimator, "sqlflow_train_loop"): sys.stderr.write( "compile keras model failed, ignoring this error " "since the model seems to defined sqlflow_train_loop.") classifier = init_model_with_feature_column( estimator, model_params, has_none_optimizer=True) has_none_optimizer = True else: six.reraise(*sys.exc_info()) train_dataset = train_dataset_fn() if val_dataset_fn is not None: validate_dataset = val_dataset_fn() else: validate_dataset = None if load: # FIXME(typhoonzero): copied from runtime.tensorflow.train_keras inputs, targets = next(iter(train_dataset.take(1))) classifier.evaluate(inputs, targets) load_keras_model_weights(classifier, load) if len(FLAGS.worker_hosts.split(",")) > 1: keras_train_distributed(classifier, model_params, save, model_meta, FLAGS, train_dataset_fn, val_dataset_fn, is_pai) else: keras_train_compiled(classifier, save, train_dataset, validate_dataset, label_meta, epochs, verbose, model_meta, validation_steps, has_none_optimizer) if is_pai: print("saving keras model to: %s" % FLAGS.sqlflow_oss_modeldir) oss.save_dir(FLAGS.sqlflow_oss_modeldir, save) oss.save_file(FLAGS.sqlflow_oss_modeldir, "model_meta.json")
def save_to_oss(self, oss_model_dir, local_dir=None): """ This save function would archive all the files on local_dir into a tarball, and save it into OSS model directory. Args: oss_model_dir (str): the OSS model directory to save. It is in the format of oss://bucket/path/to/dir/. local_dir (str): the local directory to save. Returns: None. """ if local_dir is None: local_dir = os.getcwd() with temp_file.TemporaryDirectory() as tmp_dir: tarball = os.path.join(tmp_dir, TARBALL_NAME) self._zip(local_dir, tarball) oss.save_file(oss_model_dir, tarball, TARBALL_NAME)
def save_model(model_dir, filename, model_params, train_params, feature_metas, feature_column_names, label_meta, feature_column_code): pai_model_store.save_file(model_dir, filename) pai_model_store.save_file(model_dir, "{}.pmml".format(filename)) pai_model_store.save_file(model_dir, "model_meta.json") # (TODO:lhw) remove this function call, use the new metadata in load_metas pai_model_store.save_metas( model_dir, 1, "xgboost_model_desc", "", # estimator = "" model_params, train_params, feature_metas, feature_column_names, label_meta, feature_column_code)