def get_saved_model_type_and_estimator(datasource, model_name):
"""Get oss model type and estimator name, model can be:
1. PAI ML models: model is saved by pai
2. xgboost: on OSS with model file xgboost_model_desc
3. PAI tensorflow models: on OSS with meta file: tensorflow_model_desc
Args:
datasource: the DBMS connection URI.
model_name: the model to get info
Returns:
If model is TensorFlow model, return type and estimator name
If model is XGBoost, or other PAI model, just return model type
"""
# FIXME(typhoonzero): if the model not exist on OSS, assume it's a random
# forest model should use a general method to fetch the model and see the
# model type.
meta = Model.load_metadata_from_db(datasource, model_name)
return meta.get_type(), meta.get_meta("class_name")
def submit_pai_predict(datasource,
original_sql,
select,
model,
label_name,
pred_params,
result_table,
user=""):
"""This function pack needed params and resource to a tarball
and submit a prediction task to PAI
Args:
datasource: string
Like: maxcompute://ak:[email protected]/api?
curr_project=test_ci&scheme=http
original_sql: string
Original "TO PREDICT" statement.
select: string
SQL statement to get prediction data set.
model: string
Model to load and do prediction.
label_name: string
Name of the label column, if not exist in select.
pred_params: dict
Params for training, crossponding to WITH clause.
result_table: string
The table name to save prediction result.
user: string
A string to identify the user, used to load model from the user's
directory.
"""
params = dict(locals())
# format resultTable name to "db.table" to let the codegen form a
# submitting argument of format "odps://project/tables/table_name"
project = table_ops.get_project(datasource)
if result_table.count(".") == 0:
result_table = "%s.%s" % (project, result_table)
model_metas = Model.load_metadata_from_db(datasource, model)
model_type = model_metas.get_type()
estimator = model_metas.get_meta("class_name")
setup_predict_entry(params, model_type)
train_label = model_metas.get_meta("label")
if train_label is not None:
train_label_desc = train_label.get_field_desc()[0]
else:
train_label_desc = None
if pred_params is None:
extra_result_cols = []
else:
extra_result_cols = pred_params.get("predict.extra_outputs", "")
extra_result_cols = [
c.strip() for c in extra_result_cols.split(",") if c.strip()
]
with db.connect_with_data_source(datasource) as conn:
result_column_names, train_label_idx = create_predict_table(
conn, select, result_table, train_label_desc, label_name,
extra_result_cols)
oss_model_path = pai_model.get_oss_model_save_path(datasource,
model,
user=user)
# TODO(typhoonzero): Do **NOT** create tmp table when the select statement
# is like: "SELECT fields,... FROM table"
with table_ops.create_tmp_tables_guard(select, datasource) as data_table:
del params["label_name"]
params["pai_table"] = data_table
params["result_column_names"] = result_column_names
params["train_label_idx"] = train_label_idx
params["extra_result_cols"] = extra_result_cols
if try_pai_local_run(params, oss_model_path):
return
with temp_file.TemporaryDirectory(prefix="sqlflow", dir="/tmp") as cwd:
prepare_archive(cwd, estimator, oss_model_path, params)
cmd = get_pai_predict_cmd(
datasource, project, oss_model_path, model, data_table,
result_table, model_type, pred_params,
"file://" + os.path.join(cwd, JOB_ARCHIVE_FILE),
"file://" + os.path.join(cwd, PARAMS_FILE))
submit_pai_task(cmd, datasource)
def submit_pai_explain(datasource,
original_sql,
select,
model,
model_params,
result_table,
explainer="TreeExplainer",
user=""):
"""This function pack need params and resource to a tarball
and submit a explain task to PAI
Args:
datasource: string
Like: maxcompute://ak:[email protected]/api?
curr_project=test_ci&scheme=http
original_sql: string
Original "TO PREDICT" statement.
select: string
SQL statement to get prediction data set.
model: string
Model to load and do prediction.
model_params: dict
Params for training, crossponding to WITH clause.
result_table: string
The table name to save prediction result.
user: string
A string to identify the user, used to load model from the user's
directory.
"""
params = dict(locals())
# format resultTable name to "db.table" to let the codegen form a
# submitting argument of format "odps://project/tables/table_name"
project = table_ops.get_project(datasource)
if result_table:
if result_table.count(".") == 0:
result_table = "%s.%s" % (project, result_table)
params["result_table"] = result_table
# used to save the explain image
timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
params["oss_dest"] = "explain_images/%s/%s" % (user, timestamp)
add_env_to_params(params, "SQLFLOW_OSS_AK", "oss_ak")
add_env_to_params(params, "SQLFLOW_OSS_SK", "oss_sk")
add_env_to_params(params, "SQLFLOW_OSS_ALISA_ENDPOINT", "oss_endpoint")
add_env_to_params(params, "SQLFLOW_OSS_ALISA_BUCKET", "oss_bucket_name")
meta = Model.load_metadata_from_db(datasource, model)
model_type = meta.get_type()
estimator = meta.get_meta("class_name")
label_name = model_params.get("label_col")
if label_name is None:
label_column = meta.get_meta("label")
if label_column is not None:
label_name = label_column.get_field_desc()[0].name
setup_explain_entry(params, model_type)
oss_model_path = pai_model.get_oss_model_save_path(datasource,
model,
user=user)
# TODO(typhoonzero): Do **NOT** create tmp table when the select statement
# is like: "SELECT fields,... FROM table"
with table_ops.create_tmp_tables_guard(select, datasource) as data_table:
params["pai_table"] = data_table
# Create explain result table
if result_table:
conn = db.connect_with_data_source(datasource)
feature_columns = meta.get_meta("features")
estimator_string = meta.get_meta("class_name")
field_descs = get_ordered_field_descs(feature_columns)
feature_column_names = [fd.name for fd in field_descs]
create_explain_table(conn, meta.get_type(), explainer,
estimator_string, result_table,
feature_column_names)
conn.close()
if not try_pai_local_run(params, oss_model_path):
with temp_file.TemporaryDirectory(prefix="sqlflow",
dir="/tmp") as cwd:
prepare_archive(cwd, estimator, oss_model_path, params)
cmd = get_pai_explain_cmd(
datasource, project, oss_model_path, model, data_table,
result_table, model_type, model_params,
"file://" + os.path.join(cwd, JOB_ARCHIVE_FILE),
"file://" + os.path.join(cwd, PARAMS_FILE), label_name)
submit_pai_task(cmd, datasource)
if result_table:
print('Saved result into: {}'.format(result_table))
else:
print_oss_image(params["oss_dest"], params["oss_ak"], params["oss_sk"],
params["oss_endpoint"], params["oss_bucket_name"])