parser.add_argument('--max_df',
type=float,
default=1.0,
help='the maximum document frequency.')
parser.add_argument(
'--C',
type=float,
default=1.0,
help='Inverse of regularization strength of LogisticRegression')
args = parser.parse_args()
# Polyaxon
experiment = Experiment(project='newsgroup')
experiment.create()
experiment.log_params(ngram_range=(args.ngram, args.ngram),
max_features=args.max_features,
max_df=args.max_df,
C=args.C)
# Train and eval the model with given parameters.
# Polyaxon
metrics = train_and_eval(ngram_range=(args.ngram, args.ngram),
max_features=args.max_features,
max_df=args.max_df,
C=args.C)
# Logging metrics
print("Testing metrics: {}", metrics)
# Polyaxon
experiment.log_metrics(**metrics)
type=int,
default=2000,
help='The maximum number of features.')
parser.add_argument('--max_df',
type=float,
default=1.0,
help='the maximum document frequency.')
parser.add_argument(
'--C',
type=float,
default=1.0,
help='Inverse of regularization strength of LogisticRegression')
args = parser.parse_args()
# Polyaxon
tracking.init()
# Train and eval the model with given parameters.
# Polyaxon
output_path = os.path.join(tracking.get_outputs_path(), "model.joblib")
metrics = train_and_eval(output=output_path,
ngram_range=(args.ngram, args.ngram),
max_features=args.max_features,
max_df=args.max_df,
C=args.C)
# Logging metrics
print("Testing metrics: {}", metrics)
# Polyaxon
tracking.log_metrics(**metrics)
def main():
# Clean up the model directory if not keep training
if not args.keep_train:
shutil.rmtree(args.model_dir, ignore_errors=True)
print('Remove model directory: {}'.format(args.model_dir))
if not config.LOCAL_TRAIN:
rsync_model_files = glob.glob(config.RSYNC_MODEL_DIR + "/*")
print("export models we have :", rsync_model_files)
for model_file in rsync_model_files:
model_time = int(model_file.split('/')[-1])
if model_time < int(time.time()) - 60 * 60 * 24 * 7:
print("delete :", model_file)
shutil.rmtree(model_file, ignore_errors=True)
# Clean up the model export directory
shutil.rmtree(args.model_export_dir, ignore_errors=True)
# Set Which GPU to use or do not use gpu
if args.gpu_num == '-1':
session_config = tf.ConfigProto(device_count={'CPU': args.num_threads})
else:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_num
session_config = tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True),
device_count={'CPU': args.num_threads})
# Set Model Params
model_params = {
'learning_rate':
args.learning_rate,
'layer_size_list':
list(map(lambda x: int(x), args.layer_size_list.split(","))),
'regular_rate':
args.regular_rate,
'dropout':
args.dropout,
'opt_algo':
args.opt_algo
}
estimator_config = tf.estimator.RunConfig().replace(
session_config=session_config,
log_step_count_steps=args.log_steps,
save_summary_steps=args.log_steps)
model = tf.estimator.Estimator(model_fn=model_fn,
model_dir=args.model_dir,
params=model_params,
config=estimator_config)
print("\n=========================================")
print("train type : ", args.train_type)
for embedding in config.FILE_EMBEDDING_COLUMNS:
print("{} embedding size : {}".format(embedding[0], embedding[-1]))
print("train at GPU : ", args.gpu_num)
print("learning rate : ", args.learning_rate)
print("optimize algorithm : ", args.opt_algo)
print("batch size : ", args.batch_size)
print("epochs : ", args.train_epochs)
print("layer size list : ", args.layer_size_list)
print("regular rate : ", args.regular_rate)
print("dropout rate : ", args.dropout)
print("=========================================")
print("model saved at : ", config.MODEL_EXPORT_DIR)
print("=========================================\n")
if args.train_type == 'train_and_eval':
train_and_eval(model=model,
train_data=args.train_data,
eval_data=args.eval_data,
train_epochs=args.train_epochs,
batch_size=args.batch_size,
epochs_per_eval=args.epochs_per_eval)
else:
train(model=model,
train_data=args.train_data,
train_epochs=args.train_epochs,
batch_size=args.batch_size)
export_model(model, args.model_export_dir)
'--random_state',
type=int,
default=33,
)
args = parser.parse_args()
# Polyaxon
tracking.init()
# Train and eval the model with given parameters.
# Polyaxon
model_path = "model.joblib"
metrics = train_and_eval(
model_path=model_path,
n_neighbors=args.n_neighbors,
leaf_size=args.leaf_size,
metric=args.metric,
p=args.p,
weights=args.weights,
test_size=args.test_size,
random_state=args.random_state,
)
# Logging metrics to Polyaxon
print("Testing metrics: {}", metrics)
# Polyaxon
tracking.log_metrics(**metrics)
# Logging the model
tracking.log_model(model_path, name="iris-model", framework="scikit-learn")