def load_pretrained_model_estimator(estimator,
model_params,
warm_start_from=None):
if warm_start_from is not None:
estimator_func = estimator.__init__ if inspect.isclass(
estimator) else estimator
estimator_spec = inspect.getargspec(estimator_func)
# The constructor of Estimator contains named parameter "warm_start_from"
warm_start_from_key = "warm_start_from"
if warm_start_from_key in estimator_spec.args:
warm_start_from = os.path.abspath(warm_start_from)
if is_tf_estimator(estimator):
with open("exported_path", "r") as fid:
exported_path = str(fid.read())
exported_path = os.path.abspath(exported_path)
assert exported_path.startswith(
warm_start_from), "The exported path is incorrect"
warm_start_from = exported_path
model_params[warm_start_from_key] = warm_start_from
else:
raise NotImplementedError(
"Incremental training is not supported in {}".format(
estimator))
def evaluate(datasource,
estimator_string,
select,
result_table,
feature_columns,
feature_column_names,
feature_metas={},
label_meta={},
model_params={},
validation_metrics=["Accuracy"],
save="",
batch_size=1,
validation_steps=None,
verbose=0,
pai_table=""):
FLAGS = define_tf_flags()
set_oss_environs(FLAGS)
estimator_cls = import_model(estimator_string)
is_estimator = is_tf_estimator(estimator_cls)
set_log_level(verbose, is_estimator)
is_pai = True if pai_table else False
eval_dataset = get_dataset_fn(select,
datasource,
feature_column_names,
feature_metas,
label_meta,
is_pai=is_pai,
pai_table=pai_table,
batch_size=batch_size)
model_params.update(feature_columns)
pop_optimizer_and_loss(model_params)
if is_estimator:
with open("exported_path", "r") as fid:
exported_path = str(fid.read())
model_params["warm_start_from"] = exported_path
estimator = estimator_cls(**model_params)
result_metrics = estimator_evaluate(estimator, eval_dataset,
validation_metrics)
else:
keras_model = init_model_with_feature_column(estimator_cls,
model_params)
keras_model_pkg = sys.modules[estimator_cls.__module__]
result_metrics = keras_evaluate(keras_model, eval_dataset, save,
keras_model_pkg, validation_metrics)
if result_table:
metric_name_list = ["loss"] + validation_metrics
if is_pai:
conn = PaiIOConnection.from_table(result_table)
else:
conn = db.connect_with_data_source(datasource)
write_result_metrics(result_metrics, metric_name_list, result_table,
conn)
conn.close()
def evaluate(datasource,
estimator_string,
select,
result_table,
feature_columns,
feature_column_names,
feature_metas={},
label_meta={},
model_params={},
validation_metrics=["Accuracy"],
save="",
batch_size=1,
validation_steps=None,
verbose=0,
hdfs_namenode_addr="",
hive_location="",
hdfs_user="",
hdfs_pass=""):
estimator_cls = import_model(estimator_string)
is_estimator = is_tf_estimator(estimator_cls)
set_log_level(verbose, is_estimator)
eval_dataset = get_dataset_fn(select,
datasource,
feature_column_names,
feature_metas,
label_meta,
is_pai=False,
pai_table="",
batch_size=batch_size)
model_params.update(feature_columns)
if is_estimator:
model_params["model_dir"] = save
estimator = estimator_cls(**model_params)
result_metrics = estimator_evaluate(estimator, eval_dataset,
validation_metrics)
else:
keras_model = init_model_with_feature_column(estimator_cls,
model_params)
keras_model_pkg = sys.modules[estimator_cls.__module__]
result_metrics = keras_evaluate(keras_model, eval_dataset, save,
keras_model_pkg, validation_metrics)
# write result metrics to a table
conn = connect_with_data_source(datasource)
driver = conn.driver
if result_table:
metric_name_list = ["loss"] + validation_metrics
write_result_metrics(result_metrics,
metric_name_list,
result_table,
driver,
conn,
hdfs_namenode_addr=hdfs_namenode_addr,
hive_location=hive_location,
hdfs_user=hdfs_user,
hdfs_pass=hdfs_pass)
def pred(datasource,
estimator_string,
select,
result_table,
feature_columns,
feature_column_names,
feature_column_names_map,
train_label_name,
result_col_name,
feature_metas={},
model_params={},
pred_params={},
save="",
batch_size=1,
pai_table=""):
estimator = import_model(estimator_string)
model_params.update(feature_columns)
is_estimator = is_tf_estimator(estimator)
if pai_table != "":
conn = PaiIOConnection.from_table(pai_table)
selected_cols = db.selected_cols(conn, None)
predict_generator = db.db_generator(conn, None)
else:
conn = db.connect_with_data_source(datasource)
selected_cols = db.selected_cols(conn, select)
predict_generator = db.db_generator(conn, select)
pop_optimizer_and_loss(model_params)
if pred_params is None:
extra_result_cols = []
else:
extra_result_cols = pred_params.get("extra_outputs", "")
extra_result_cols = [
c.strip() for c in extra_result_cols.split(",") if c.strip()
]
if not is_estimator:
if not issubclass(estimator, tf.keras.Model):
# functional model need field_metas parameter
model_params["field_metas"] = feature_metas
print("Start predicting using keras model...")
keras_predict(estimator, model_params, save, result_table,
feature_column_names, feature_metas, train_label_name,
result_col_name, conn, predict_generator, selected_cols,
extra_result_cols)
else:
# TODO(sneaxiy): support extra_result_cols for estimator
model_params['model_dir'] = save
print("Start predicting using estimator model...")
estimator_predict(result_table, feature_column_names, feature_metas,
train_label_name, result_col_name, conn,
predict_generator, selected_cols)
print("Done predicting. Predict table : %s" % result_table)
def pred(datasource,
estimator_string,
select,
result_table,
feature_columns,
feature_column_names,
feature_column_names_map,
train_label_name,
result_col_name,
feature_metas={},
model_params={},
save="",
batch_size=1,
hdfs_namenode_addr="",
hive_location="",
hdfs_user="",
hdfs_pass=""):
estimator = import_model(estimator_string)
model_params.update(feature_columns)
is_estimator = is_tf_estimator(estimator)
conn = db.connect_with_data_source(datasource)
driver = conn.driver
predict_generator = db.db_generator(conn, select)
selected_cols = db.selected_cols(conn, select)
if not is_estimator:
if not issubclass(estimator, tf.keras.Model):
# functional model need field_metas parameter
model_params["field_metas"] = feature_metas
print("Start predicting using keras model...")
keras_predict(estimator, model_params, save, result_table,
feature_column_names, feature_metas, train_label_name,
result_col_name, driver, conn, predict_generator,
selected_cols, hdfs_namenode_addr, hive_location,
hdfs_user, hdfs_pass)
else:
model_params['model_dir'] = save
print("Start predicting using estimator model...")
estimator_predict(estimator, model_params, save, result_table,
feature_column_names, feature_column_names_map,
feature_columns, feature_metas, train_label_name,
result_col_name, driver, conn, predict_generator,
selected_cols, hdfs_namenode_addr, hive_location,
hdfs_user, hdfs_pass)
print("Done predicting. Predict table : %s" % result_table)
def pred(datasource,
estimator_string,
select,
result_table,
feature_columns,
feature_column_names,
feature_column_names_map,
train_label_name,
result_col_name,
feature_metas={},
model_params={},
save="",
batch_size=1,
hdfs_namenode_addr="",
hive_location="",
hdfs_user="",
hdfs_pass="",
is_pai=False,
pai_table=""):
# import custom model package
runtime.import_model_def(estimator_string, globals())
estimator = eval(estimator_string)
model_params.update(feature_columns)
is_estimator = is_tf_estimator(estimator)
if not is_estimator:
if not issubclass(estimator, tf.keras.Model):
# functional model need field_metas parameter
model_params["field_metas"] = feature_metas
print("Start predicting using keras model...")
keras_predict(estimator, model_params, save, result_table, is_pai,
pai_table, feature_column_names, feature_metas,
train_label_name, result_col_name, datasource, select,
hdfs_namenode_addr, hive_location, hdfs_user, hdfs_pass)
else:
model_params['model_dir'] = save
print("Start predicting using estimator model...")
estimator_predict(estimator, model_params, save, result_table,
feature_column_names, feature_column_names_map,
feature_columns, feature_metas, train_label_name,
result_col_name, datasource, select,
hdfs_namenode_addr, hive_location, hdfs_user,
hdfs_pass, is_pai, pai_table)
print("Done predicting. Predict table : %s" % result_table)
def train(datasource,
estimator_string,
select,
validation_select,
feature_columns,
feature_column_names,
feature_metas={},
label_meta={},
model_params={},
validation_metrics=["Accuracy"],
save="",
batch_size=1,
epoch=1,
validation_steps=1,
verbose=0,
max_steps=None,
validation_start_delay_secs=0,
validation_throttle_secs=0,
save_checkpoints_steps=100,
log_every_n_iter=10,
load_pretrained_model=False,
is_pai=True,
pai_table="",
pai_val_table="",
feature_columns_code="",
model_params_code_map={},
model_repo_image="",
original_sql="",
feature_column_names_map=None):
# TODO(sneaxiy): collect features and label
model_meta = collect_metadata(original_sql=original_sql,
select=select,
validation_select=validation_select,
model_repo_image=model_repo_image,
class_name=estimator_string,
attributes=model_params,
features=None,
label=None)
estimator = import_model(estimator_string)
is_estimator = is_tf_estimator(estimator)
if verbose < 1: # always use verbose == 1 when using PAI to get more logs
verbose = 1
set_log_level(verbose, is_estimator)
model_params.update(feature_columns)
FLAGS = define_tf_flags()
set_oss_environs(FLAGS)
num_workers = len(FLAGS.worker_hosts.split(","))
worker_id = FLAGS.task_index
train_dataset_fn = get_dataset_fn(select,
datasource,
feature_column_names,
feature_metas,
label_meta,
is_pai,
pai_table,
batch_size,
epochs=epoch,
shuffle_size=1000,
num_workers=num_workers,
worker_id=worker_id)
val_dataset_fn = None
if validation_select:
val_dataset_fn = get_dataset_fn(validation_select, datasource,
feature_column_names, feature_metas,
label_meta, is_pai, pai_val_table,
batch_size)
if not is_estimator:
if isinstance(estimator, types.FunctionType):
# functional model need field_metas parameter
model_params["field_metas"] = feature_metas
keras_train_and_save(estimator, model_params, save, FLAGS,
train_dataset_fn, val_dataset_fn, label_meta,
epoch, verbose, validation_metrics,
validation_steps, load_pretrained_model,
model_meta)
else:
estimator_train_and_save(estimator, model_params, save, FLAGS,
train_dataset_fn, val_dataset_fn,
log_every_n_iter, max_steps,
validation_start_delay_secs,
validation_throttle_secs,
save_checkpoints_steps, validation_metrics,
load_pretrained_model, model_meta)
# save model to OSS
if num_workers == 1 or worker_id == 0:
oss_model_dir = FLAGS.sqlflow_oss_modeldir
oss.save_oss_model(oss_model_dir, estimator_string, is_estimator,
feature_column_names, feature_column_names_map,
feature_metas, label_meta, model_params_code_map,
feature_columns_code, num_workers)
print("Model saved to oss: %s" % oss_model_dir)
print("Done training")
def train(datasource,
estimator_string,
select,
validation_select,
feature_columns,
feature_column_names,
feature_metas={},
label_meta={},
model_params={},
validation_metrics=["Accuracy"],
save="",
batch_size=1,
epoch=1,
validation_steps=1,
verbose=0,
max_steps=None,
validation_start_delay_secs=0,
validation_throttle_secs=0,
save_checkpoints_steps=100,
log_every_n_iter=10,
load_pretrained_model=False,
is_pai=False,
pai_table="",
pai_val_table="",
feature_columns_code="",
model_params_code_map={},
model_repo_image="",
original_sql="",
feature_column_names_map=None):
# NOTE(typhoonzero): feature_column_names_map is used only for PAI
# submitter API.
# TODO(sneaxiy): collect features and label
model_meta = collect_metadata(original_sql=original_sql,
select=select,
validation_select=validation_select,
model_repo_image=model_repo_image,
class_name=estimator_string,
attributes=model_params,
features=None,
label=None)
estimator = import_model(estimator_string)
is_estimator = is_tf_estimator(estimator)
set_log_level(verbose, is_estimator)
model_params.update(feature_columns)
train_dataset_fn = get_dataset_fn(select,
datasource,
feature_column_names,
feature_metas,
label_meta,
is_pai,
pai_table,
batch_size,
epochs=epoch,
shuffle_size=1000)
val_dataset_fn = None
if validation_select:
val_dataset_fn = get_dataset_fn(validation_select, datasource,
feature_column_names, feature_metas,
label_meta, is_pai, pai_val_table,
batch_size)
if not is_estimator: # keras
if isinstance(estimator, types.FunctionType):
# functional model need field_metas parameter
model_params["field_metas"] = feature_metas
keras_train_and_save(estimator, model_params, save, is_pai,
train_dataset_fn, val_dataset_fn, label_meta,
epoch, verbose, validation_metrics,
validation_steps, load_pretrained_model,
model_meta)
else:
estimator_train_and_save(estimator, model_params, save,
train_dataset_fn, val_dataset_fn, max_steps,
validation_start_delay_secs,
validation_throttle_secs,
save_checkpoints_steps, validation_metrics,
load_pretrained_model, model_meta)
# remove cache files
any(map(os.remove, glob.glob('cache_train.*')))
any(map(os.remove, glob.glob('cache_validation.*')))
print("Done training")
def evaluate(datasource,
estimator_string,
select,
result_table,
feature_columns,
feature_column_names,
feature_metas={},
label_meta={},
model_params={},
validation_metrics=["Accuracy"],
save="",
batch_size=1,
validation_steps=None,
verbose=0,
hdfs_namenode_addr="",
hive_location="",
hdfs_user="",
hdfs_pass="",
is_pai=False,
pai_table=""):
# import custom model package
runtime.import_model_def(estimator_string, globals())
estimator_cls = eval(estimator_string)
is_estimator = is_tf_estimator(estimator_cls)
set_log_level(verbose, is_estimator)
eval_dataset, _ = get_dataset_fn(select,
"",
datasource,
feature_column_names,
feature_metas,
label_meta,
is_pai,
pai_table,
"",
1,
batch_size,
1,
is_estimator=is_estimator)
model_params.update(feature_columns)
if is_estimator:
if is_pai:
FLAGS = tf.app.flags.FLAGS
model_params["model_dir"] = FLAGS.checkpointDir
else:
model_params["model_dir"] = save
# tf estimator always have feature_column argument
estimator = estimator_cls(**model_params)
result_metrics = estimator_evaluate(estimator, eval_dataset,
validation_metrics)
else:
keras_model = init_model_with_feature_column(estimator, model_params)
keras_model_pkg = sys.modules[estimator_cls.__module__]
result_metrics = keras_evaluate(keras_model, eval_dataset, save,
keras_model_pkg, validation_metrics)
# write result metrics to a table
if is_pai:
driver = "pai_maxcompute"
conn = None
else:
conn = connect_with_data_source(datasource)
driver = conn.driver
if result_table:
metric_name_list = ["loss"] + validation_metrics
write_result_metrics(result_metrics,
metric_name_list,
result_table,
driver,
conn,
hdfs_namenode_addr=hdfs_namenode_addr,
hive_location=hive_location,
hdfs_user=hdfs_user,
hdfs_pass=hdfs_pass)
def train(datasource,
estimator_string,
select,
validation_select,
feature_columns,
feature_column_names,
feature_metas={},
label_meta={},
model_params={},
validation_metrics=["Accuracy"],
save="",
batch_size=1,
epoch=1,
validation_steps=1,
verbose=0,
max_steps=None,
validation_start_delay_secs=0,
validation_throttle_secs=0,
save_checkpoints_steps=100,
log_every_n_iter=10,
load_pretrained_model=False,
is_pai=False,
pai_table="",
pai_val_table="",
feature_columns_code="",
model_repo_image="",
original_sql=""):
model_meta = collect_model_metadata(original_sql, select,
validation_select, estimator_string,
model_params, feature_columns_code,
feature_metas, label_meta, None,
model_repo_image)
# import custom model package
runtime.import_model_def(estimator_string, globals())
estimator = eval(estimator_string)
is_estimator = is_tf_estimator(estimator)
if is_pai and verbose < 1: # always use verbose == 1 when using PAI to get more logs
verbose = 1
set_log_level(verbose, is_estimator)
# fill in feature columns parameters
model_params.update(feature_columns)
FLAGS = None
num_workers = 1
worker_id = 0
# only support distributed training on PAI (TF version 1.x)
if is_pai:
FLAGS = define_tf_flags()
set_oss_environs(FLAGS)
num_workers = len(FLAGS.worker_hosts.split(","))
worker_id = FLAGS.task_index
# TODO(typhoonzero): remove this after update the keras models.
# copy feature_name to name field for Keras functional models:
# https://github.com/sql-machine-learning/models/blob/develop/sqlflow_models/dnnclassifier_functional_api_example.py
for k in feature_metas:
feature_metas[k]["name"] = feature_metas[k]["feature_name"]
train_dataset_fn, val_dataset_fn = get_dataset_fn(
select,
validation_select,
datasource,
feature_column_names,
feature_metas,
label_meta,
is_pai,
pai_table,
pai_val_table,
epoch,
batch_size,
1000,
num_workers=num_workers,
worker_id=worker_id,
is_estimator=is_estimator)
if not is_estimator: # keras
if isinstance(estimator, types.FunctionType):
# functional model need field_metas parameter
model_params["field_metas"] = feature_metas
keras_train_and_save(estimator, model_params, save, is_pai, FLAGS,
train_dataset_fn, val_dataset_fn, label_meta,
epoch, verbose, validation_metrics,
validation_steps, load_pretrained_model,
model_meta)
else:
estimator_train_and_save(estimator, model_params, save, is_pai, FLAGS,
train_dataset_fn, val_dataset_fn,
log_every_n_iter, max_steps,
validation_start_delay_secs,
validation_throttle_secs,
save_checkpoints_steps, validation_metrics,
load_pretrained_model, model_meta)
# remove cache files
any(map(os.remove, glob.glob('cache_train.*')))
any(map(os.remove, glob.glob('cache_validation.*')))
print("Done training")
def train_step(original_sql,
model_image,
estimator_string,
datasource,
select,
validation_select,
model_params,
train_params,
validation_params,
feature_column_map,
label_column,
save,
load=None,
pai_table=None,
pai_val_table=None):
if model_params is None:
model_params = {}
if train_params is None:
train_params = {}
if validation_params is None:
validation_params = {}
if load:
Model.load_from_db(datasource, load)
load = "model_save"
else:
load = None
is_pai = True if pai_table else False
fc_map = compile_ir_feature_columns(feature_column_map,
EstimatorType.TENSORFLOW)
field_descs = get_ordered_field_descs(feature_column_map)
feature_column_names = [fd.name for fd in field_descs]
feature_metas = dict([(fd.name, fd.to_dict(dtype_to_string=True))
for fd in field_descs])
# no label for clustering model
label_meta = None
if label_column:
label_meta = label_column.get_field_desc()[0].to_dict(
dtype_to_string=True)
feature_column_names_map = dict()
for target in feature_column_map:
fclist = feature_column_map[target]
feature_column_names_map[target] = [
fc.get_field_desc()[0].name for fc in fclist
]
# Construct optimizer objects to pass to model initializer.
# The original model_params is serializable (do not have tf.xxx objects).
model_params_constructed = copy.deepcopy(model_params)
for optimizer_arg in ["optimizer", "dnn_optimizer", "linear_optimizer"]:
if optimizer_arg in model_params_constructed:
model_params_constructed[optimizer_arg] = get_tf_optimizer(
model_params_constructed[optimizer_arg])
if "loss" in model_params_constructed:
model_params_constructed["loss"] = get_tf_loss(
model_params_constructed["loss"])
# extract params for training.
verbose = train_params.get("verbose", 1)
batch_size = train_params.get("batch_size", 1)
epoch = train_params.get("epoch", 1)
save_checkpoints_steps = train_params.get("save_checkpoints_steps", 100)
max_steps = train_params.get("max_steps", None)
if max_steps is not None and max_steps <= 0:
max_steps = None
validation_metrics = validation_params.get("metrics", "Accuracy")
validation_metrics = [v.strip() for v in validation_metrics.split(",")]
validation_steps = validation_params.get("steps", 1)
validation_start_delay_secs = validation_params.get("start_delay_secs", 0)
validation_throttle_secs = validation_params.get("throttle_secs", 0)
estimator = import_model(estimator_string)
is_estimator = is_tf_estimator(estimator)
# always use verbose == 1 when using PAI to get more logs
if verbose < 1:
verbose = 1
set_log_level(verbose, is_estimator)
model_params_constructed.update(fc_map)
FLAGS = define_tf_flags()
set_oss_environs(FLAGS)
num_workers = len(FLAGS.worker_hosts.split(","))
worker_id = FLAGS.task_index
train_dataset_fn = get_dataset_fn(select,
datasource,
feature_column_names,
feature_metas,
label_meta,
is_pai,
pai_table,
batch_size,
epochs=epoch,
shuffle_size=1000,
num_workers=num_workers,
worker_id=worker_id)
val_dataset_fn = None
if validation_select or pai_val_table:
val_dataset_fn = get_dataset_fn(validation_select, datasource,
feature_column_names, feature_metas,
label_meta, is_pai, pai_val_table,
batch_size)
model_meta = collect_metadata(original_sql=original_sql,
select=select,
validation_select=validation_select,
model_repo_image=model_image,
class_name=estimator_string,
attributes=model_params,
features=feature_column_map,
label=label_column)
# FIXME(typhoonzero): avoid save model_meta twice, keras_train_and_save,
# estimator_train_and_save also dumps model_meta to a file under cwd.
# should only keep the model.save_to_db part.
save_dir = "model_save"
if not is_estimator:
if isinstance(estimator, types.FunctionType):
# functional model need field_metas parameter
model_params_constructed["field_metas"] = feature_metas
keras_train_and_save(estimator, model_params_constructed, save_dir,
FLAGS, train_dataset_fn, val_dataset_fn,
label_meta, epoch, verbose, validation_metrics,
validation_steps, load, model_meta, is_pai)
else:
estimator_train_and_save(estimator, model_params_constructed, save_dir,
FLAGS, train_dataset_fn, val_dataset_fn,
max_steps, validation_start_delay_secs,
validation_throttle_secs,
save_checkpoints_steps, validation_metrics,
load, model_meta)
# save model to DB/OSS
model = Model(EstimatorType.TENSORFLOW, model_meta)
if num_workers == 1 or worker_id == 0:
saved = model.save_to_db(datasource,
save,
oss_model_dir=FLAGS.sqlflow_oss_modeldir)
print("Model saved to DB: %s" % saved)
print("Done training")
def predict_step(datasource,
select,
result_table,
label_name,
model,
pai_table=None):
if isinstance(model, six.string_types):
model = Model.load_from_db(datasource, model)
else:
assert isinstance(model,
Model), "not supported model type %s" % type(model)
model_params = model.get_meta("attributes")
train_fc_map = model.get_meta("features")
label_meta = model.get_meta("label")
train_label_desc = label_meta.get_field_desc()[0] if label_meta else None
train_label_name = train_label_desc.name if train_label_desc else None
estimator_string = model.get_meta("class_name")
save = "model_save"
field_descs = get_ordered_field_descs(train_fc_map)
feature_column_names = [fd.name for fd in field_descs]
feature_metas = dict([(fd.name, fd.to_dict(dtype_to_string=True))
for fd in field_descs])
feature_columns = compile_ir_feature_columns(train_fc_map,
model.get_type())
is_pai = True if pai_table else False
if is_pai:
select = "SELECT * FROM %s" % pai_table
conn = db.connect_with_data_source(datasource)
result_column_names, train_label_idx = create_predict_table(
conn, select, result_table, train_label_desc, label_name)
if is_pai:
conn.close()
conn = PaiIOConnection.from_table(pai_table)
select = None
selected_cols = result_column_names[0:-1]
if train_label_idx >= 0:
selected_cols = selected_cols[0:train_label_idx] + [
train_label_name
] + selected_cols[train_label_idx:]
estimator = import_model(estimator_string)
model_params.update(feature_columns)
is_estimator = is_tf_estimator(estimator)
predict_generator = db.db_generator(conn, select)
pop_optimizer_and_loss(model_params)
if not is_estimator:
if not issubclass(estimator, tf.keras.Model):
# functional model need field_metas parameter
model_params["field_metas"] = feature_metas
print("Start predicting using keras model...")
keras_predict(estimator, model_params, save, result_table,
feature_column_names, feature_metas, train_label_name,
label_name, conn, predict_generator, selected_cols)
else:
model_params['model_dir'] = save
print("Start predicting using estimator model...")
estimator_predict(result_table, feature_column_names, feature_metas,
train_label_name, label_name, conn,
predict_generator, selected_cols)
print("Done predicting. Predict table : %s" % result_table)
conn.close()
