def main():
configs = json.load(open('train_config.json', 'r'))
mlflow.set_experiment(configs['experiment_name'])
with mlflow.start_run():
# Set MLFlow logs
mlflow.keras.autolog()
mlflow.log_param('loss', configs['loss'])
mlflow.log_param('epochs', configs['epochs'])
mlflow.log_param('batch_size', configs['batch_size'])
mlflow.log_param('data_path', configs['data_filepath'])
# Instantiates Keras callbacks
model_save_path = ".saved_models/model.h5"
checkpoint = ModelCheckpoint(model_save_path,
monitor='val_loss',
mode='min',
save_best_only='True',
verbose=1)
# Preprocesses data
processor = DataProcessor(configs['data_filepath'])
samples_train, samples_test, labels_train, labels_test, snr_train, snr_test = processor.process(
)
# DEFINE YOUR MODEL HERE:
model = Sequential()
model.add(Dense(32, activation='relu', input_shape=(512, )))
model.add(Dense(4, activation='softmax'))
model.compile(loss=configs['loss'],
optimizer=configs['optimizer'],
metrics=['accuracy'])
plot_path = '.model_plot.png'
plot_model(model,
to_file=plot_path,
show_shapes=True,
show_layer_names=True)
# Trains model
model.fit(samples_train,
labels_train,
batch_size=configs['batch_size'],
epochs=configs['epochs'],
verbose=0,
validation_split=0.2,
callbacks=[checkpoint])
mlflow.log_artifact(model_save_path)
mlflow.log_artifact(plot_path)
os.remove(model_save_path)
os.remove(plot_path)
# Tests model and scores
user_input = input(
"Do you want test your model On the Test Set? (y/n) ")
if user_input == 'y':
score = model.evaluate(samples_test, labels_test, verbose=0)
print('Test loss', score[0])
print('Test accuracy', score[1])
def test_search_runs():
mlflow.set_experiment("exp-for-search")
# Create a run and verify that the current active experiment is the one we just set
logged_runs = {}
with mlflow.start_run() as active_run:
logged_runs["first"] = active_run.info.run_id
mlflow.log_metric("m1", 0.001)
mlflow.log_metric("m2", 0.002)
mlflow.log_metric("m1", 0.002)
mlflow.log_param("p1", "a")
mlflow.set_tag("t1", "first-tag-val")
with mlflow.start_run() as active_run:
logged_runs["second"] = active_run.info.run_id
mlflow.log_metric("m1", 0.008)
mlflow.log_param("p2", "aa")
mlflow.set_tag("t2", "second-tag-val")
def verify_runs(runs, expected_set):
assert set([r.info.run_id for r in runs
]) == set([logged_runs[r] for r in expected_set])
experiment_id = MlflowClient().get_experiment_by_name(
"exp-for-search").experiment_id
# 2 runs in this experiment
assert len(MlflowClient().list_run_infos(experiment_id,
ViewType.ACTIVE_ONLY)) == 2
# 2 runs that have metric "m1" > 0.001
runs = MlflowClient().search_runs([experiment_id], "metrics.m1 > 0.0001")
verify_runs(runs, ["first", "second"])
# 1 run with has metric "m1" > 0.002
runs = MlflowClient().search_runs([experiment_id], "metrics.m1 > 0.002")
verify_runs(runs, ["second"])
# no runs with metric "m1" > 0.1
runs = MlflowClient().search_runs([experiment_id], "metrics.m1 > 0.1")
verify_runs(runs, [])
# 1 run with metric "m2" > 0
runs = MlflowClient().search_runs([experiment_id], "metrics.m2 > 0")
verify_runs(runs, ["first"])
# 1 run each with param "p1" and "p2"
runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'",
ViewType.ALL)
verify_runs(runs, ["first"])
runs = MlflowClient().search_runs([experiment_id], "params.p2 != 'a'",
ViewType.ALL)
verify_runs(runs, ["second"])
runs = MlflowClient().search_runs([experiment_id], "params.p2 = 'aa'",
ViewType.ALL)
verify_runs(runs, ["second"])
# 1 run each with tag "t1" and "t2"
runs = MlflowClient().search_runs([experiment_id],
"tags.t1 = 'first-tag-val'",
ViewType.ALL)
verify_runs(runs, ["first"])
runs = MlflowClient().search_runs([experiment_id], "tags.t2 != 'qwerty'",
ViewType.ALL)
verify_runs(runs, ["second"])
runs = MlflowClient().search_runs([experiment_id],
"tags.t2 = 'second-tag-val'",
ViewType.ALL)
verify_runs(runs, ["second"])
# delete "first" run
MlflowClient().delete_run(logged_runs["first"])
runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'",
ViewType.ALL)
verify_runs(runs, ["first"])
runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'",
ViewType.DELETED_ONLY)
verify_runs(runs, ["first"])
runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'",
ViewType.ACTIVE_ONLY)
verify_runs(runs, [])
from pyspark.ml.classification import * from pyspark.ml.evaluation import * from pyspark.sql.functions import * from pyspark.ml.tuning import CrossValidator, ParamGridBuilder import warnings # Mlflow libaries import mlflow from mlflow import spark # In[81]: # Set experiment # This will actually automatically create one if the one you call on doesn't exist mlflow.set_experiment(experiment_name="Experiment-3") # set up your client from mlflow.tracking import MlflowClient client = MlflowClient() # In[137]: # Create a run and attach it to the experiment you just created experiments = client.list_experiments( ) # returns a list of mlflow.entities.Experiment experiment_name = "Experiment-3" def create_run(experiment_name):
def train_fcn(exp_name=None,
model_name=None,
penalty=None,
alpha=None,
l1_ratio=None,
learning_rate=None,
n_estimators=None,
max_depth=None):
"""
# Training the model with the given parameters
Parameters :
exp_name : Choose from "Compare_algo" or "Best_algo".
Compare_algo is for comparing the differents model and best algo is the best selected algo
model_name : Choose from 'Linear Regression', 'SGD Regression', 'RF Regression', 'DT Regression',
'GBDT Regression', 'XGB Regression'
Parameters for Linear Regression : None
Parameters for SGD Regression : penalty, alpha, l1_ratio,
learning_rate (constant, optimal, invscaling, adaptive)
Parameters for RF Regression : n_estimators (int), max_depth (int)
Parameters for DT Regression : max_depth (int)
Parameters for GBDT Regression : n_estimators (int), max_depth (int), learning_rate (float)
Parameters for XGB Regression : n_estimators (int), max_depth (int), learning_rate (float)
"""
# Create an experiment - "Compare_algo" or "Best_algo"
exp_name = exp_name
mlflow.set_experiment(exp_name)
# Getting the data
X_train, X_test, Y_train, Y_test = load_data()
path = "./temp_images/"
# Useful for multiple runs
with mlflow.start_run():
# Execute the model - Selecting the model based on the model name
model = models_selection(model_name, penalty, alpha, l1_ratio,
learning_rate, n_estimators, max_depth)
model.fit(X_train, Y_train)
# Evaluation
predictions = model.predict(X_test)
rmse, mae, r2 = eval_metrics(Y_test, predictions)
# Print metrics
print("Model trained")
print("Metric values :")
print(rmse, mae, r2)
# Creating a plot which will be tracked as an artifact
# Calculating the errors
delta_y = Y_test - predictions
sns.distplot(np.array(delta_y), label='ERROR (ΔY)')
plt.xlabel('Errors : ΔY')
plt.ylabel('Density')
plt.title('Distribution of the Errors')
plt.savefig(path + '/error_dist.png')
# Log_parameters, Metrics and Model to MLFlow
mlflow.log_params({
"penalty": penalty,
"alpha": alpha,
"l1_ratio": l1_ratio,
"learning_rate": learning_rate,
"n_estimators": n_estimators,
"max_depth": max_depth
})
mlflow.log_metrics({"rmse": rmse, "mae": mae, "r2": r2})
mlflow.log_artifact(path + '/error_dist.png')
# print("Saved to {}".format(mlflow.get_artifact_uri()))
# Load model
mlflow.sklearn.log_model(model, model_name)
# returning the metric value
return r2
import mlflow
remote_server_uri = "http://127.0.0.1:5000" # set to your server URI
mlflow.set_tracking_uri(remote_server_uri)
# Note: on Databricks, the experiment name passed to mlflow_set_experiment must be a
# valid path in the workspace
mlflow.set_experiment("/my-first-experiment")
with mlflow.start_run():
mlflow.log_param("a", 1)
mlflow.log_metric("b", 2)
def test_log_batch_handler_success(mock_get_request_message,
mock_get_request_json, tmpdir):
# Test success cases for the LogBatch API
def _test_log_batch_helper_success(metric_entities,
param_entities,
tag_entities,
expected_metrics=None,
expected_params=None,
expected_tags=None):
"""
Simulates a LogBatch API request using the provided metrics/params/tags, asserting that it
succeeds & that the backing store contains either the set of expected metrics/params/tags
(if provided) or, by default, the metrics/params/tags used in the API request.
"""
with mlflow.start_run() as active_run:
run_id = active_run.info.run_uuid
mock_get_request_message.return_value = LogBatch(
run_id=run_id,
metrics=[m.to_proto() for m in metric_entities],
params=[p.to_proto() for p in param_entities],
tags=[t.to_proto() for t in tag_entities])
response = _log_batch()
assert response.status_code == 200
json_response = json.loads(response.get_data())
assert json_response == {}
_assert_logged_entities(run_id, expected_metrics
or metric_entities, expected_params
or param_entities, expected_tags
or tag_entities)
store = FileStore(tmpdir.strpath)
mock_get_request_json.return_value = "{}" # Mock request JSON so it passes length validation
server_patch = mock.patch('mlflow.server.handlers._get_store',
return_value=store)
client_patch = mock.patch('mlflow.tracking.utils._get_store',
return_value=store)
with server_patch, client_patch:
mlflow.set_experiment("log-batch-experiment")
# Log an empty payload
_test_log_batch_helper_success([], [], [])
# Log multiple metrics/params/tags
_test_log_batch_helper_success(metric_entities=[
Metric(key="m-key", value=3.2 * i, timestamp=i) for i in range(3)
],
param_entities=[
Param(key="p-key-%s" % i,
value="p-val-%s" % i)
for i in range(4)
],
tag_entities=[
RunTag(key="t-key-%s" % i,
value="t-val-%s" % i)
for i in range(5)
])
# Log metrics with the same key
_test_log_batch_helper_success(metric_entities=[
Metric(key="m-key", value=3.2 * i, timestamp=3) for i in range(3)
],
param_entities=[],
tag_entities=[])
# Log tags with the same key, verify the last one gets written
same_key_tags = [
RunTag(key="t-key", value="t-val-%s" % i) for i in range(5)
]
_test_log_batch_helper_success(metric_entities=[],
param_entities=[],
tag_entities=same_key_tags,
expected_tags=[same_key_tags[-1]])
N_ESTIMATORS = 2
MAX_DEPTH = 2
model = RandomForestRegressor(n_estimators=N_ESTIMATORS, max_depth=MAX_DEPTH)
model = model.fit(x_train, y_train.values.ravel())
# save model
joblib.dump(model, 'models/model.joblib')
joblib.dump(column_order, 'models/column_order.joblib')
if settings.SHOULD_USE_MLFLOW == 'true':
# log training run to mlflow
uri = f'http://{settings.MLFLOW_IP}:5000'
print("uri", uri)
mlflow.set_tracking_uri(uri=uri)
if settings.CI == 'true':
mlflow.set_experiment('CI')
else:
mlflow.set_experiment('dev')
with mlflow.start_run() as run:
# calculate evaluation metrics
y_test_pred = model.predict(x_test)
rmse = sqrt(
metrics.mean_squared_error(y_true=y_test, y_pred=y_test_pred))
r2_score = metrics.r2_score(y_true=y_test, y_pred=y_test_pred)
# log hyperparameters to mlflow
mlflow.log_param('n_estimators', N_ESTIMATORS)
mlflow.log_param('max_depth', MAX_DEPTH)
mlflow.log_param('n_columns_training_data', len(x.columns))
validation_loader = None
if args.test_data:
logging.info("Building test dataset")
test_dataset = MeliChallengeDataset(dataset_path=args.test_data,
random_buffer_size=1)
test_loader = DataLoader(test_dataset,
batch_size=128,
shuffle=False,
collate_fn=pad_sequences,
drop_last=False)
else:
test_dataset = None
test_loader = None
mlflow.set_experiment(f"diplodatos.{args.language}")
with mlflow.start_run():
logging.info("Starting experiment")
# Log all relevent hyperparameters
mlflow.log_params({
"model_type": "CNN",
"embeddings": args.pretrained_embeddings,
"dropout": args.dropout,
"embeddings_size": args.embeddings_size,
"epochs": args.epochs
})
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
logging.info("Building classifier")
train_df, test_df = df.randomSplit([.8, .2], seed=142)
assembler = VectorAssembler(handleInvalid='skip',
inputCols=['TV', 'Radio', 'Newspaper'],
outputCol='features')
estimator = GBTRegressor(featuresCol='features',
labelCol='Sales',
predictionCol='prediction')
pipeline_obj = Pipeline(stages=[assembler, estimator])
mlflow.set_tracking_uri('http://localhost:5000/')
print("mlflow tracking_uri: " + mlflow.tracking.get_tracking_uri())
mlflow.set_experiment("Advertising Regression Online Lesson")
with mlflow.start_run(run_name="spark-advertising-gbt-regressor") as run:
# Log params:
mlflow.log_param("min_info_gain", estimator.getMinInfoGain())
mlflow.log_param("max_depth", estimator.getMaxDepth())
mlflow.log_param("max_bins", estimator.getMaxBins())
mlflow.log_param("step_size", estimator.getStepSize())
# Log the model while training
pipelineModel = pipeline_obj.fit(train_df)
mlflow.spark.log_model(pipelineModel, "model")
# Log metrics: RMSE and R2
predDF = pipelineModel.transform(test_df)
regressionEvaluator = RegressionEvaluator(predictionCol="prediction",
def main():
t = Timer()
with t.timer(f'fix seed RANDOM_STATE:{RANDOM_STATE}'):
seed_everything(RANDOM_STATE)
with t.timer(f'read label'):
y_train = pd.read_csv(f'{INPUT_DIR}/train.solution',
header=None).T.values[0].reshape(-1, 1)
if LABEL_LOG_SCALING is True:
with t.timer(f'label log scaling (log->mms[0, 1]'):
scaler, y_train = label_scaling(y_train)
with t.timer(f'read features'):
unique_num_dic = {}
feature_index = {}
X_train = pd.DataFrame()
X_valid = pd.DataFrame()
X_test = pd.DataFrame()
fidx = 0
for feat in dense_features:
logging.info(f'[dense][{feat}] read feature ...')
feature_index[feat] = fidx
fidx += 1
X_train = pd.concat([
X_train,
pd.read_feather(f'{FEATURE_DIR}/{feat}_train.feather')
],
axis=1)
X_valid = pd.concat([
X_valid,
pd.read_feather(f'{FEATURE_DIR}/{feat}_valid.feather')
],
axis=1)
X_test = pd.concat([
X_test,
pd.read_feather(f'{FEATURE_DIR}/{feat}_test.feather')
],
axis=1)
for feat in sparse_features:
logging.info(f'[sparse][{feat}] read feature ...')
feature_index[feat] = fidx
fidx += 1
X_train = pd.concat([
X_train,
pd.read_feather(f'{FEATURE_DIR}/{feat}_train.feather')
],
axis=1)
X_valid = pd.concat([
X_valid,
pd.read_feather(f'{FEATURE_DIR}/{feat}_valid.feather')
],
axis=1)
X_test = pd.concat([
X_test,
pd.read_feather(f'{FEATURE_DIR}/{feat}_test.feather')
],
axis=1)
unique_num = pd.concat(
[X_train[feat], X_valid[feat], X_test[feat]]).nunique()
unique_num_dic[feat] = unique_num
for feat in varlen_sparse_features:
logging.info(f'[varlen sparse][{feat}] read feature ...')
feature_index[feat] = (fidx, fidx + VARLEN_MAX_LEN)
fidx += VARLEN_MAX_LEN
train_feat = pd.read_feather(
f'{FEATURE_DIR}/{feat}_train.feather').values
varlen_list = [i[0] for i in train_feat]
varlen_list = pad_sequences(
varlen_list,
maxlen=VARLEN_MAX_LEN,
padding='post',
)
X_train = pd.concat([X_train, pd.DataFrame(varlen_list)], axis=1)
valid_feat = pd.read_feather(
f'{FEATURE_DIR}/{feat}_valid.feather').values
varlen_list = [i[0] for i in valid_feat]
varlen_list = pad_sequences(
varlen_list,
maxlen=VARLEN_MAX_LEN,
padding='post',
)
X_valid = pd.concat([X_valid, pd.DataFrame(varlen_list)], axis=1)
test_feat = pd.read_feather(
f'{FEATURE_DIR}/{feat}_test.feather').values
varlen_list = [i[0] for i in test_feat]
varlen_list = pad_sequences(
varlen_list,
maxlen=VARLEN_MAX_LEN,
padding='post',
)
X_test = pd.concat([X_test, pd.DataFrame(varlen_list)], axis=1)
tmp = []
for i in [i[0]
for i in train_feat] + [i[0] for i in valid_feat
] + [i[0] for i in test_feat]:
tmp.extend(i)
unique_num = len(set(tmp))
unique_num_dic[feat] = unique_num
X_train = X_train.fillna(0.0)
X_valid = X_valid.fillna(0.0)
X_test = X_test.fillna(0.0)
with t.timer(f'READ folds'):
folds = pd.read_csv(
f'{FOLD_DIR}/train_folds_{FOLD_NAME}{FOLD_NUM}_RS{RANDOM_STATE}.csv'
)
with t.timer(f'inference by saved models'):
preds_train_val_models = pd.DataFrame()
preds_valid_models = pd.DataFrame()
preds_test_models = pd.DataFrame()
for run_idx, (EXP_NAME, run_id) in enumerate(ENSEMBLE_MODELS.items()):
mlflow.set_experiment(EXP_NAME)
run = mlflow.get_run(run_id)
MODEL_NAME = run.data.params['model']
BATCH_SIZE = int(run.data.params['batch_size'])
DNN_HIDDEN_UNITS = [
int(i)
for i in ast.literal_eval(run.data.params['dnn_hidden_layer'])
]
DNN_DROPOUT = float(run.data.params['dnn_dropout'])
DNN_ACTIVATION = run.data.params['dnn_activation']
L2_REG = float(run.data.params['l2_reg'])
INIT_STD = float(run.data.params['init_std'])
SPAESE_EMBEDDING_DIM = int(run.data.params['embedding_dim'])
VARLEN_MODE_LIST = ast.literal_eval(
run.data.params['varlen_mode_list'])
preds_train_val = np.zeros(len(folds[folds.kfold != -1]))
preds_valid = pd.DataFrame()
preds_test = pd.DataFrame()
for fold_idx in range(FOLD_NUM):
val_idx = folds[folds.kfold == fold_idx].index.tolist()
x_val = X_train.iloc[val_idx]
y_val = y_train[val_idx]
model = load_model(
MODEL_NAME,
BATCH_SIZE,
DNN_HIDDEN_UNITS,
DNN_DROPOUT,
DNN_ACTIVATION,
L2_REG,
INIT_STD,
SPAESE_EMBEDDING_DIM,
VARLEN_MODE_LIST,
feature_index=feature_index,
unique_num_dic=unique_num_dic,
)
weight_path = f'{WEIFGHT_DIR}/train_weights_mlflow-{run_id}_fold{fold_idx}.h5'
model.load_state_dict(torch.load(weight_path))
preds_train_val_fold = model.predict(x_val, BATCH_SIZE)
preds_train_val[val_idx] = preds_train_val_fold
preds_val_inv = label_inverse_scaling(
scaler, preds_train_val_fold.reshape(-1, 1))
y_val_inv = label_inverse_scaling(scaler, y_val)
val_metric = mean_squared_log_error(y_val_inv, preds_val_inv)
logging.info(f'{EXP_NAME}, FOLD{fold_idx}, MSLE:{val_metric}')
preds_valid_fold = model.predict(X_valid, BATCH_SIZE)
preds_valid[f'preds_{fold_idx}'] = preds_valid_fold
preds_test_fold = model.predict(X_test, BATCH_SIZE)
preds_test[f'preds_{fold_idx}'] = preds_test_fold
val_idx = folds[folds.kfold != -1].index.tolist()
preds_train_val_inv = label_inverse_scaling(
scaler, preds_train_val.reshape(-1, 1))
y_train_inv = label_inverse_scaling(scaler, y_train[val_idx])
cv = mean_squared_log_error(y_train_inv, preds_train_val_inv)
logging.info(f'RUN_ID:{run_id}, CV-MSLE:{cv}')
preds_valid_mean = preds_valid[[
f'preds_{fold_idx}' for fold_idx in range(FOLD_NUM)
]].mean(axis=1).values
preds_valid_mean_scaled = label_inverse_scaling(
scaler, preds_valid_mean.reshape(-1, 1))
preds_valid['preds_log'] = preds_valid_mean
preds_valid['preds'] = preds_valid_mean_scaled
preds_test_mean = preds_test[[
f'preds_{fold_idx}' for fold_idx in range(FOLD_NUM)
]].mean(axis=1).values
preds_test_mean_scaled = label_inverse_scaling(
scaler, preds_test_mean.reshape(-1, 1))
preds_test['preds_log'] = preds_test_mean
preds_test['preds'] = preds_test_mean_scaled
preds_train_val_models[
f'preds_{EXP_NAME}'] = preds_train_val_inv.reshape(-1)
preds_valid_models[f'preds_{EXP_NAME}'] = preds_valid['preds']
preds_test_models[f'preds_{EXP_NAME}'] = preds_test['preds']
preds_train_val_models[f'preds_log_{EXP_NAME}'] = preds_train_val
preds_valid_models[f'preds_log_{EXP_NAME}'] = preds_valid[
'preds_log']
preds_test_models[f'preds_log_{EXP_NAME}'] = preds_test[
'preds_log']
for run_idx, (EXP_NAME, run_id) in enumerate(ENSEMBLE_MODELS.items()):
save_as_feather(f'preds_log_{EXP_NAME}',
f'{SAVE_DIR}/model_predict',
preds_train_val_models, preds_valid_models,
preds_test_models)
with t.timer(f'make each submission'):
for model_preds_log_col in [
f'preds_log_{EXP_NAME}' for EXP_NAME in ENSEMBLE_MODELS.keys()
]:
preds_log_valid_models = preds_valid_models[[model_preds_log_col]]
preds_valid_models_mean = preds_log_valid_models.mean(
axis=1).values
preds_valid_models_mean_scaled = label_inverse_scaling(
scaler, preds_valid_models_mean.reshape(-1, 1))
preds_log_valid_models['preds_log'] = preds_valid_models_mean
preds_log_valid_models['preds'] = preds_valid_models_mean_scaled
preds_log_valid_models = post_process(preds_log_valid_models)
sub_name = model_preds_log_col
make_submission('validation', preds_log_valid_models, sub_name)
preds_log_test_models = preds_test_models[[model_preds_log_col]]
preds_test_models_mean = preds_log_test_models.mean(axis=1).values
preds_test_models_mean_scaled = label_inverse_scaling(
scaler, preds_test_models_mean.reshape(-1, 1))
preds_log_test_models['preds_log'] = preds_test_models_mean
preds_log_test_models['preds'] = preds_test_models_mean_scaled
preds_log_test_models = post_process(preds_log_test_models)
sub_name = model_preds_log_col
make_submission('test', preds_log_test_models, sub_name)
with t.timer(f'blend mean'):
model_preds_log_col = [
f'preds_log_{EXP_NAME}' for EXP_NAME in ENSEMBLE_MODELS.keys()
]
preds_log_valid_models = preds_valid_models[model_preds_log_col]
preds_valid_models_mean = preds_log_valid_models.mean(axis=1).values
preds_valid_models_mean_scaled = label_inverse_scaling(
scaler, preds_valid_models_mean.reshape(-1, 1))
preds_log_valid_models['preds_log'] = preds_valid_models_mean
preds_log_valid_models['preds'] = preds_valid_models_mean_scaled
preds_log_valid_models = post_process(preds_log_valid_models)
sub_name = 'Blend__' + '__'.join(
[f'{EXP_NAME}' for EXP_NAME in ENSEMBLE_MODELS.keys()])
make_submission('validation', preds_log_valid_models, sub_name)
model_preds_log_col = [
f'preds_log_{EXP_NAME}' for EXP_NAME in ENSEMBLE_MODELS.keys()
]
preds_log_test_models = preds_test_models[model_preds_log_col]
preds_test_models_mean = preds_log_test_models.mean(axis=1).values
preds_test_models_mean_scaled = label_inverse_scaling(
scaler, preds_test_models_mean.reshape(-1, 1))
preds_log_test_models['preds_log'] = preds_test_models_mean
preds_log_test_models['preds'] = preds_test_models_mean_scaled
preds_log_test_models = post_process(preds_log_test_models)
sub_name = 'Blend__' + '__'.join(
[f'{EXP_NAME}' for EXP_NAME in ENSEMBLE_MODELS.keys()])
make_submission('test', preds_log_test_models, sub_name)
with t.timer(f'stacking (Ridge)'):
model_preds_log_col = [
f'preds_log_{EXP_NAME}' for EXP_NAME in ENSEMBLE_MODELS.keys()
]
preds_log_train_models = preds_train_val_models[model_preds_log_col]
preds_log_valid_models = preds_valid_models[model_preds_log_col]
preds_log_test_models = preds_test_models[model_preds_log_col]
train_val_idx = folds[folds.kfold != -1].index.tolist()
smodel = Ridge(alpha=1.0)
smodel_name = 'ridge'
stack_train, stack_valid, stack_test = predict_cv(
smodel,
smodel_name,
preds_log_train_models,
y_train[train_val_idx].reshape(-1),
preds_log_valid_models,
preds_log_test_models,
n_splits=5)
stack_valid_scaled = label_inverse_scaling(scaler,
stack_valid.reshape(-1, 1))
preds_log_valid_models['preds'] = stack_valid_scaled
preds_log_valid_models['preds_log'] = stack_valid
preds_log_valid_models = post_process(preds_log_valid_models)
sub_name = 'Ridge__' + '__'.join(
[f'{EXP_NAME}' for EXP_NAME in ENSEMBLE_MODELS.keys()])
make_submission('validation', preds_log_valid_models, sub_name)
stack_test_scaled = label_inverse_scaling(scaler,
stack_test.reshape(-1, 1))
preds_log_test_models['preds'] = stack_test_scaled
preds_log_test_models['preds_log'] = stack_test
preds_log_test_models = post_process(preds_log_test_models)
sub_name = 'Ridge__' + '__'.join(
[f'{EXP_NAME}' for EXP_NAME in ENSEMBLE_MODELS.keys()])
make_submission('test', preds_log_test_models, sub_name)
"IPython.notebook.kernel.execute(`notebookName = '${IPython.notebook.notebook_name}'`);"
)
# In[22]:
notebookName
# In[25]:
# remove joblib artifacts from MLBOX_SAVE, these are not needed
shutil.rmtree(os.path.join(MLBOX_SAVE, 'joblib'))
# In[26]:
# save sample as mlflow artifact
experiment_id = mlflow.set_experiment('hyperparms')
#%%
with mlflow.start_run(experiment_id=experiment_id,
run_name='mlbox_inital_hyperparm'):
mlflow.log_param('notebook_name', notebookName)
mlflow.log_artifacts(MLBOX_SAVE)
# ## Clean-up
# In[17]:
shutil.rmtree(TMPDIR)
# In[ ]:
from keras.models import Sequential, load_model
from keras.layers.core import Dense, Dropout, Activation
from keras.utils import np_utils
# get_ipython().run_line_magic('matplotlib', 'inline')
# In[2]:
if 0:
get_ipython().system('pip install mlflow')
import mlflow
mlflow.tracking.set_tracking_uri(r'http://127.0.0.1:5000')
print('start tracking on: ',mlflow.tracking.get_tracking_uri())
import datetime
exp_id = mlflow.set_experiment('exp-'+ '15') #str(datetime.datetime.now().second))
print(f'start experiment {exp_id}')
# In[3]:
def build_model():
# building a linear stack of layers with the sequential model
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(512))
model.add(Activation('relu'))
def _reset_experiment(): # this is needed because the active experiment in mlflow is sticky, so if # an earlier test sets this, then things fail here when we don't use # an explicit experiment mlf.set_experiment(mlf.entities.Experiment.DEFAULT_EXPERIMENT_NAME)
learning_rate = 0.3 * (batch_size * cumulation) / 256
optimizer = LARS(
model.parameters(),
lr=learning_rate,
weight_decay=0.0005,
exclude_from_weight_decay=["batch_normalization", "bias"],
device=device
)
# "decay the learning rate with the cosine decay schedule without restarts"
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, 100, eta_min=0, last_epoch=-1
)
mlflow.set_experiment("SimCLR")
with mlflow.start_run(run_name="SimCLR test") as run:
mlflow.log_params(params_ml_flow)
loss_array = []
# Training
for epoch in range(100):
print(f"EPOCH - {epoch+1}")
cpt = 1
epoch_size = len(dataloader)
model.train()
for xi, xj in dataloader:
import click
import tensorflow as tf
import tensorflow.keras.backend as K
#from tensorflow.python.keras.layers.core import Dropout
from classifier import setup_ds
from pathlib import Path
import joblib
import mlflow
mlflow.set_experiment("hydra-digits")
from models import block
from tensorflow.keras import Input, Model
from tensorflow.keras.layers import AvgPool2D, Flatten, Dense, Softmax, ReLU, Dropout
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from icecream import ic
import numpy as np
from contextlib import redirect_stdout
def log_model(model, name):
file_path = f"/tmp/{name}.txt"
with open(file_path, 'w') as f:
with redirect_stdout(f):
model.summary()
mlflow.log_artifact(file_path)
def head(x, n_classes, dropout_ratio=0.5, dense_nodes=[128, 128], name=None):
input_ = Input(K.int_shape(x)[1:])
y = input_
# TODO learn about droupout across time
def run_clustering(*,
config_path: str,
alg: str,
dest_path: str,
n_clusters_max: int,
use_mlflow: bool,
experiment_name: str = None,
run_name: str = None):
"""
Run clustering for data specified in the experiment configuration.
:param config_path: Path to the configuration experiment file.
It should be located in "cfg" directory and contain necessary keys,
which specify the target dataset names and base path.
:param alg: Type of algorithm to utilize for clustering.
Possible options are: "km" for K-Means and "gm" for Gaussian Mixture
Model respectively.
:param dest_path: Destination path where all results will be stored.
:param n_clusters_max: Maximum number of clusters to run on
all images. Each will be clustered with the number of centres in range
from two to "n_clusters_max".
:param use_mlflow: Boolean indicating whether to utilize mlflow.
:param experiment_name: Name of the experiment. Used only if
use_mlflow = True.
:param run_name: Name of the run. Used only if use_mlflow = True.
"""
if use_mlflow:
args = locals()
mlflow.set_tracking_uri("http://beetle.mlflow.kplabs.pl")
mlflow.set_experiment(experiment_name)
mlflow.start_run(run_name=run_name)
log_params_to_mlflow(args)
log_tags_to_mlflow(args['run_name'])
with open(config_path, 'r') as setup_file:
setup = yaml.safe_load(setup_file)
images = setup['exp_cfg']['train_ids'] + setup['exp_cfg']['test_ids']
os.makedirs(dest_path, exist_ok=True)
print('Running clustering...')
for img_name, img_id in images:
img_base_path = Path(setup['exp_cfg']['dpath']) / img_name / img_id
gt = envi.open(list(img_base_path.glob("*_fixedmask.hdr"))[0])
gt = np.array(gt.open_memmap(), dtype=np.int)
gt = np.where(gt > 128, 1, 0)
img = np.array(
load_img(list(img_base_path.glob('*_B8.TIF'))[0],
color_mode='grayscale',
target_size=gt.shape))
mask = np.where(img != BACKGROUND_LABEL)
data = np.expand_dims(img[mask].ravel(), -1)
y_true = gt[mask].ravel()
img_dest_path = os.path.join(dest_path, f'{img_name}-{img_id}')
os.makedirs(img_dest_path, exist_ok=True)
metrics = {}
for n_clusters in range(2, n_clusters_max + 1):
y_pred = CLUSTERS[alg](n_clusters,
random_state=0).fit_predict(data)
metrics[f'{n_clusters}-clusters'] = {
key: f(labels_true=y_true, labels_pred=y_pred)
for key, f in METRICS.items()
}
predicted_map = np.full(img.shape, -1)
predicted_map[mask] = y_pred
np.savetxt(os.path.join(img_dest_path,
f'{n_clusters}-predicted-map.txt'),
predicted_map,
fmt='%i')
imsave(
os.path.join(img_dest_path, f'{n_clusters}-predicted-map.png'),
img_as_ubyte(label2rgb(predicted_map)))
gt_map = np.full(img.shape, -1)
gt_map[mask] = y_true
np.savetxt(os.path.join(img_dest_path, f'ground-truth-map.txt'),
gt_map,
fmt='%i')
imsave(os.path.join(img_dest_path, f'ground-truth-map.png'),
img_as_ubyte(label2rgb(gt_map)))
Image.fromarray(img).save(
os.path.join(img_dest_path, 'original-map.png'))
with open(os.path.join(img_dest_path, 'metrics.json'),
'w') as metrics_file:
json.dump(metrics, metrics_file, ensure_ascii=False, indent=4)
print(metrics)
if use_mlflow:
mlflow.log_artifacts(dest_path, artifact_path=dest_path)
shutil.rmtree(dest_path)
import mlflow
import os
import time
from mlflow import log_metric, log_param, log_artifact
if __name__ == "__main__":
mlflow.set_experiment("First")
with mlflow.start_run():
# Log a parameter (key-value pair)
log_param("param1", 5)
# Log a metric; metrics can be updated throughout the run
for i in range(200):
time.sleep(0.1)
log_metric("foo1", 1 * i)
log_metric("foo2", 2 * i)
log_metric("foo3", 3 * i)
log_metric("foo4", 3 * i)
log_metric("foo5", 3 * i)
log_metric("foo6", 3 * i)
log_metric("foo7", 3 * i)
log_metric("foo8", 3 * i)
log_metric("foo9", 3 * i)
log_metric("foo10", 3 * i)
log_metric("foo11", 3 * i)
log_metric("foo12", 3 * i)
log_metric("foo13", 3 * i)
log_metric("foo14", 3 * i)
log_metric("foo15", 3 * i)
log_metric("foo16", 3 * i)
def main(epochs, batch_size, lr, device, train_reversed, label_smoothing,
weight_decay, validation_interval, mlflow_experiment_name,
embedding_dim, embedding_height, embedding_width, input_channels,
output_channels, kernel_height, kernel_width, embedding_dropout,
feature_map_dropout, projection_dropout):
"""
This script trains a ConvE KGE model and validates the model every 10 epochs
(--validation_interval).
If the MRR score has improved since the last checkpoint a new checkpoint
will be logged to the mlflow server.
"""
mappings, datasets = load()
train = datasets['train']
valid = datasets['valid']
test = datasets['test']
id2rel = mappings['id2rel']
id2e = mappings['id2e']
num_entities = len(id2e)
num_relations = len(id2rel)
mlflow.set_experiment(mlflow_experiment_name)
mlflow.start_run()
mlflow.log_param('Epochs', epochs)
mlflow.log_param('Label Smoothing', label_smoothing)
mlflow.log_param('train_reversed', train_reversed)
mlflow.log_param('Batch Size', batch_size)
mlflow.log_param('Learning Rate', lr)
mlflow.log_param('Weight Decay', weight_decay)
mlflow.log_param('num_entities', num_entities)
mlflow.log_param('num_relations', num_relations)
mlflow.log_param('embedding_dim', embedding_dim)
mlflow.log_param('ConvE_input_channels', input_channels)
mlflow.log_param('ConvE_output_channels', output_channels)
mlflow.log_param('ConvE_width', embedding_width)
mlflow.log_param('ConvE_height', embedding_height)
mlflow.log_param('ConvE_kernel_height', kernel_height)
mlflow.log_param('ConvE_kernel_width', kernel_width)
mlflow.log_param('ConvE_input_dropout', embedding_dropout)
mlflow.log_param('ConvE_feature_map_dropout', feature_map_dropout)
mlflow.log_param('ConvE_output_dropout', projection_dropout)
model = get_model(num_entities, num_relations, device)
model.compile(metrics=[
HitsAtK(1),
HitsAtK(3),
HitsAtK(10),
MeanRank(),
MeanReciprocalRank()
],
callbacks=[
MlFlowLogger(mlflow),
SaveModel('mean_reciprocal_rank', objective='max')
])
try:
losses, _ = model.fit(train,
valid,
learning_rate=lr,
num_epochs=epochs,
train_reversed=train_reversed,
label_smoothing=label_smoothing,
weight_decay=weight_decay,
validation_interval=validation_interval,
batch_size=batch_size)
except KeyboardInterrupt:
logger.warning('Forced Keyboard Interrupt. Exiting now...')
mlflow.log_param('training_interrupted', True)
sys.exit()
epochs = len(losses)
mlflow.log_param('Early Stop Epochs', epochs)
for epoch in range(epochs):
mlflow.log_metric('loss', losses[epoch], step=epoch)
logger.info("*" * 30)
logger.info("Evaluating on Test set")
logger.info("*" * 30)
with torch.no_grad():
results = model.evaluate(test, train)
for epoch, item in enumerate(results.items()):
mlflow.log_metric(item[0], item[1], step=epoch)
mlflow.pytorch.log_model(model, 'models/conve-model-final')
mlflow.end_run()
model.eval()
h = torch.tensor(test[0, 0:1], dtype=torch.long, device=device)
p = torch.tensor(test[0, 1:2], dtype=torch.long, device=device)
t = torch.tensor(test[0, 1:2], dtype=torch.long, device=device)
obj_scores = model.score_objects(h, p).detach()
subj_scores = model.score_subjects(p, t).detach()
print('Object Scores')
print(f'(min/max): ({obj_scores.min()}, {obj_scores.max()})')
o_sort, o_args = torch.sort(obj_scores, descending=True)
print('Sorted Top 10 Scores: ', o_sort[:10])
print('Sorted Top 10 Ids: ', o_args[:10])
print('True Id: ', t.item())
print('Subject Scores')
print(f'(min/max): ({subj_scores.min()}, {subj_scores.max()})')
s_sort, s_args = torch.sort(subj_scores, descending=True)
print('Sorted Top 10 Scores: ', s_sort[:10])
print('Sorted Top 10 Ids: ', s_args[:10])
print('True Id: ', h.item())
def train(in_alpha, in_l1_ratio):
def eval_metrics(actual, pred):
rmse = np.sqrt(mean_squared_error(actual, pred))
mae = mean_absolute_error(actual, pred)
r2 = r2_score(actual, pred)
return rmse, mae, r2
warnings.filterwarnings("ignore")
np.random.seed(40)
# Read the wine-quality csv file (make sure you're running this from the root of MLflow!)
# Assumes wine-quality.csv is located in the same folder as the notebook
wine_path = "wine-quality.csv"
data = pd.read_csv(wine_path)
# Split the data into training and test sets. (0.75, 0.25) split.
train, test = train_test_split(data)
# The predicted column is "quality" which is a scalar from [3, 9]
train_x = train.drop(["quality"], axis=1)
test_x = test.drop(["quality"], axis=1)
train_y = train[["quality"]]
test_y = test[["quality"]]
# Set default values if no alpha is provided
if float(in_alpha) is None:
alpha = 0.5
else:
alpha = float(in_alpha)
# Set default values if no l1_ratio is provided
if float(in_l1_ratio) is None:
l1_ratio = 0.5
else:
l1_ratio = float(in_l1_ratio)
mlflow.set_experiment('mlflow_demo')
# Useful for multiple runs (only doing one run in this sample notebook)
with mlflow.start_run():
# Execute ElasticNet
lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42)
lr.fit(train_x, train_y)
# Evaluate Metrics
predicted_qualities = lr.predict(test_x)
(rmse, mae, r2) = eval_metrics(test_y, predicted_qualities)
# Print out metrics
print("Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio))
print(" RMSE: %s" % rmse)
print(" MAE: %s" % mae)
print(" R2: %s" % r2)
# Log parameter, metrics, and model to MLflow
mlflow.log_param("alpha", alpha)
mlflow.log_param("l1_ratio", l1_ratio)
mlflow.log_metric("rmse", rmse)
mlflow.log_metric("r2", r2)
mlflow.log_metric("mae", mae)
mlflow.sklearn.log_model(lr, "model_wine")
import mlflow
mlflow.set_experiment('rec_sys')
mlflow.start_run(nested=True)
import comm
import baseline
if __name__ == "__main__":
# comm.main()
baseline.main("offline_train")
baseline.main("evaluate")
# baseline.main("online_train")
# baseline.main("submit")
import os
import shutil
from random import random, randint
import mlflow
from mlflow import log_metric, log_param, log_artifacts
if __name__ == "__main__":
# set the tracking server to be Databricks Community Edition
# set the experiment name; if name does not exist, MLflow will
# create one for you
mlflow.set_tracking_uri("databricks")
mlflow.set_experiment("/Users/[email protected]/Jules_CE_Test")
print("Running experiment_ce.py")
print("Tracking on https://community.cloud.databricks.com")
mlflow.start_run(run_name="CE_TEST")
log_param("param-1", randint(0, 100))
log_metric("metric-1", random())
log_metric("metric-2", random() + 1)
log_metric("metric-3", random() + 2)
if not os.path.exists("outputs"):
os.makedirs("outputs")
with open("outputs/test.txt", "w") as f:
f.write("Looks, like I logged on the Community Edition!")
log_artifacts("outputs")
shutil.rmtree('outputs')
mlflow.end_run()
import warnings
import sys
import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
from sklearn.model_selection import train_test_split
from sklearn.linear_model import ElasticNet
from urllib.parse import urlparse
import mlflow
import mlflow.sklearn
import logging
logging.basicConfig(level=logging.WARN)
logger = logging.getLogger(__name__)
mlflow.set_experiment('wine_quality_model_experiment')
def eval_metrics(actual, pred):
rmse = np.sqrt(mean_squared_error(actual, pred))
mae = mean_absolute_error(actual, pred)
r2 = r2_score(actual, pred)
return rmse, mae, r2
if __name__ == "__main__":
warnings.filterwarnings("ignore")
np.random.seed(40)
# Read the wine-quality csv file from the URL
csv_url = (
"http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv"
def initial_values(self):
state_0, bparam_0 = self.initial_value()
state_1 = tree_map(lambda a: a - 0.08, state_0)
states = [state_0, state_1]
bparam_1 = tree_map(lambda a: a + 0.05, bparam_0)
bparams = [bparam_0, bparam_1]
return states, bparams
if __name__ == "__main__":
problem = DataContClassifier()
with open(problem.HPARAMS_PATH, "r") as hfile:
hparams = json.load(hfile)
mlflow.set_tracking_uri(hparams['meta']["mlflow_uri"])
mlflow.set_experiment(hparams['meta']["name"])
with mlflow.start_run(run_name=hparams['meta']["method"] + "-" +
hparams["meta"]["optimizer"]) as run:
ae_params, bparam = problem.initial_value()
bparam = pytree_element_add(bparam, 0.0)
mlflow.log_dict(hparams, artifact_file="hparams/hparams.json")
artifact_uri = mlflow.get_artifact_uri()
print("Artifact uri: {}".format(artifact_uri))
mlflow.log_text("", artifact_file="output/_touch.txt")
artifact_uri2 = mlflow.get_artifact_uri("output/")
print("Artifact uri: {}".format(artifact_uri2))
hparams["meta"]["output_dir"] = artifact_uri2
file_name = f"{artifact_uri2}/version.jsonl"
sw = StateWriter(file_name=file_name)
# MAGIC * Log the model as MLeap flavor # MAGIC * Save the data schema as an artifact # COMMAND ---------- # MAGIC %md ### Setup # COMMAND ---------- # MAGIC %run ../Common # COMMAND ---------- import mlflow import mlflow.mleap mlflow.set_experiment(experiment_name_python) experiment_name_python # COMMAND ---------- # MAGIC %md ### Read data # COMMAND ---------- data = spark.read.csv(data_path, header="true", inferSchema="true") (trainingData, testData) = data.randomSplit([0.7, 0.3], 2019) # COMMAND ---------- # MAGIC %md ### Train pipeline and log to MLflow
# imports
import mlflow
from pathlib import Path
from azureml.core import Workspace
# get workspace
ws = Workspace.from_config()
prefix = Path(__file__).parent
# project settings
project_uri = str(prefix.joinpath("src"))
# azure ml settings
experiment_name = "pytorch-mnist-mlproject-example"
compute_name = "gpu-cluster"
# setup mlflow tracking
mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri())
mlflow.set_experiment(experiment_name)
# setup backend config
backend_config = {"COMPUTE": compute_name, "USE_CONDA": False}
# run mlflow project
run = mlflow.projects.run(
uri=project_uri, backend="azureml", backend_config=backend_config
)
def main():
args = read_args()
dataset, dev_dataset, test_dataset = load_dataset(args.dataset_dir,
args.batch_size)
nlabels = dataset[TARGET_COL].unique().shape[0]
# It's important to always use the same one-hot length
one_hot_columns = {
one_hot_col: dataset[one_hot_col].max()
for one_hot_col in ['Gender', 'Color1']
}
embedded_columns = {
embedded_col: dataset[embedded_col].max() + 1
for embedded_col in ['Breed1']
}
numeric_columns = ['Age', 'Fee']
# TODO (optional) put these three types of columns in the same dictionary with "column types"
X_train, y_train = process_features(dataset, one_hot_columns,
numeric_columns, embedded_columns)
direct_features_input_shape = (X_train['direct_features'].shape[1], )
X_dev, y_dev = process_features(dev_dataset, one_hot_columns,
numeric_columns, embedded_columns)
# Create the tensorflow Dataset
batch_size = 32
# TODO shuffle the train dataset!
train_ds = tf.data.Dataset.from_tensor_slices(
(X_train, y_train)).batch(batch_size)
dev_ds = tf.data.Dataset.from_tensor_slices(
(X_dev, y_dev)).batch(batch_size)
test_ds = tf.data.Dataset.from_tensor_slices(
process_features(test_dataset,
one_hot_columns,
numeric_columns,
embedded_columns,
test=True)[0]).batch(batch_size)
# TODO: Build the Keras model
# model = ....
# TODO: Fit the model
mlflow.set_experiment(args.experiment_name)
with mlflow.start_run(nested=True):
# Log model hiperparameters first
mlflow.log_param('hidden_layer_size', args.hidden_layer_sizes)
mlflow.log_param('embedded_columns', embedded_columns)
mlflow.log_param('one_hot_columns', one_hot_columns)
# mlflow.log_param('numerical_columns', numerical_columns) # Not using these yet
mlflow.log_param('epochs', args.epochs)
# Train
# history = model.fit(train_ds, epochs=args.epochs)
# TODO: analyze history to see if model converges/overfits
# TODO: Evaluate the model, calculating the metrics.
# Option 1: Use the model.evaluate() method. For this, the model must be
# already compiled with the metrics.
# performance = model.evaluate(X_test, y_test)
loss, accuracy = 0, 0
# loss, accuracy = model.evaluate(dev_ds)
print("*** Dev loss: {} - accuracy: {}".format(loss, accuracy))
mlflow.log_metric('loss', loss)
mlflow.log_metric('accuracy', accuracy)
# Option 2: Use the model.predict() method and calculate the metrics using
# sklearn. We recommend this, because you can store the predictions if
# you need more analysis later. Also, if you calculate the metrics on a
# notebook, then you can compare multiple classifiers.
predictions = 'No prediction yet'
# predictions = model.predict(test_ds)
# TODO: Convert predictions to classes
# TODO: Save the results for submission
# ...
print(predictions)
print("\tcand=%s, select_method=%s" % (args.cand, args.select_method))
print("\tnum_init_labeled = %d" % (args.lab_start))
print("\ttau = %1.6f, gamma = %1.6f, delta = %1.6f, h = %1.6f" %
(args.tau, args.gamma, args.delta, args.h))
print("\tnumber of runs = {}".format(args.runs))
print("\n\n")
ans = input("Do you want to proceed with this test?? [y/n] ")
while ans not in ['y', 'n']:
ans = input("Sorry, please input either 'y' or 'n'")
if ans == 'n':
print("Not running test, exiting...")
else:
client = mlflow.tracking.MlflowClient()
#experiment_name = 'checker2'
mlflow.set_experiment(args.experiment_name)
experiment = client.get_experiment_by_name(args.experiment_name)
for i, seed in enumerate(j**2 + 3 for j in range(args.runs)):
print("=======================================")
print("============= Run {}/{} ===============".format(
i + 1, args.runs))
print("=======================================")
np.random.seed(seed)
init_labeled, unlabeled = train_test_split(
np.arange(N), train_size=2, stratify=labels
) #list(np.random.choice(range(N), 10, replace=False))
init_labeled, unlabeled = list(init_labeled), list(unlabeled)
params_shared = {
'init_labeled': init_labeled,
study_name = "adam-hyper-balanced" + f"_seed={seed}"
study_file = study_name + ".pkl"
study_filepath = os.path.join(run_params["PATH_PREFIX"], "optuna_studies",
study_file)
run_optimize = True
# Create the optuna study which shares the experiment name
if os.path.exists(study_filepath):
study = joblib.load(study_filepath)
else:
study = optuna.create_study(study_name=study_name,
direction="maximize")
if run_optimize:
experiment_id = mlflow.set_experiment(study_name)
mlflow.fastai.autolog()
# Propagate logs to the root logger.
optuna.logging.set_verbosity(verbosity=optuna.logging.INFO)
try:
study.optimize(
lambda trial: optimize(
trial, experiment_id, seed=seed, data_seed=data_seed),
n_trials=10,
callbacks=[lambda study, trial: clean_memory()],
)
except (RuntimeError, KeyboardInterrupt) as e:
print(e)
def workflow(epochs, kernel_sizes, width, height, dict_path, n_words,
duplicates, data_zipfile, region, execution_role_arn,
instance_type, app_name, model_uri):
next_id = get_next_experiment_id(
mlflow.tracking.MlflowClient().list_experiments())
next_exp = "Captcha #{}".format(next_id)
mlflow.create_experiment(next_exp)
mlflow.set_experiment(next_exp)
with mlflow.start_run(
): # probably bug? (without it nesting does not work on first run in a new experiment)
pass
if data_zipfile == 'None': # workaround for not specifying
print("No data zipfile. Trying to generate data.")
run_entrypoint(
"generate", {
"width": width,
"height": height,
"dict_path": dict_path,
"n_words": n_words,
"duplicates": duplicates,
"output_dir": "output"
})
else:
# with mlflow.start_run(run_name="load_data") as active_run:
run_entrypoint("load_data", {"data_zipfile": data_zipfile})
with mlflow.start_run(run_name="Hyperparameter tuning") as active_run:
# captcha = os.path.join(load_data_run.info.artifact_uri, os.path.splitext(data_zipfile)[0])
if data_zipfile:
captcha = os.path.splitext(data_zipfile)[0]
else:
captcha = "output"
if model_uri != "None":
kernel_sizes = [-1] # kernels doesnt matter in retraining mode
for kernel_size in [int(kernel) for kernel in kernel_sizes.split()]:
run_entrypoint(
"train", {
"epochs": epochs,
"kernel_size": kernel_size,
"width": width,
"height": height,
"dict_path": dict_path,
"n_words": n_words,
"duplicates": duplicates,
"data_dir": captcha,
"model_uri": model_uri
})
client = MlflowClient()
runs = client.search_runs([str(next_id)],
"tags.mlflow.parentRunId = '{}' ".format(
active_run.info.run_id))
best_run = max(runs, key=lambda r: r.data.metrics["test_accuracy"])
mlflow.set_tag("best_run", best_run.info.run_id)
mlflow.log_metrics(
{"best_accuracy": best_run.data.metrics["test_accuracy"]})
print("best_run", best_run)
model_path = os.path.join(best_run.info.artifact_uri, "model",
"model.h5")
run_entrypoint("convert", {"model_path": model_path})
run_entrypoint(
"deploy", {
"run_id": best_run.info.run_id,
"region": region,
"execution_role_arn": execution_role_arn,
"instance_type": instance_type,
"app_name": app_name
})
def run_train_cv(self) -> None:
"""クロスバリデーションでの学習・評価を行う
学習・評価とともに、各foldのモデルの保存、スコアのログ出力についても行う
"""
# mlflow
mlflow.set_experiment(self.exp_name)
mlflow.start_run(run_name=self.run_name)
logger.info(f"{self.run_name} - start training cv")
scores = []
va_idxes = []
preds = []
# Adversarial validation
if self.advanced and "adversarial_validation" in self.advanced:
X_train = self.X_train
X_test = self.X_test
X_train["target"] = 0
X_test["target"] = 1
X_train = pd.concat([X_train, X_test],
sort=False).reset_index(drop=True)
y_train = X_train["target"]
X_train.drop("target", axis=1, inplace=True)
X_test.drop("target", axis=1, inplace=True)
self.X_train = X_train
self.y_train = y_train
# 各foldで学習を行う
for i_fold in range(self.cv.n_splits):
# 学習を行う
logger.info(f"{self.run_name} fold {i_fold} - start training")
model, va_idx, va_pred, score = self.train_fold(i_fold)
logger.info(
f"{self.run_name} fold {i_fold} - end training - score {score}"
)
# モデルを保存する
model.save_model()
# 結果を保持する
va_idxes.append(va_idx)
scores.append(score)
preds.append(va_pred)
# 各foldの結果をまとめる
va_idxes = np.concatenate(va_idxes)
order = np.argsort(va_idxes)
preds = np.concatenate(preds, axis=0)
preds = preds[order]
if self.evaluation_metric == "log_loss":
cv_score = log_loss(self.y_train, preds, eps=1e-15, normalize=True)
elif self.evaluation_metric == "mean_absolute_error":
cv_score = mean_absolute_error(self.y_train, preds)
elif self.evaluation_metric == "rmse":
cv_score = np.sqrt(mean_squared_error(self.y_train, preds))
elif self.evaluation_metric == "auc":
cv_score = roc_auc_score(self.y_train, preds)
elif self.evaluation_metric == "prauc":
cv_score = average_precision_score(self.y_train, preds)
logger.info(f"{self.run_name} - end training cv - score {cv_score}")
# 予測結果の保存
Data.dump(preds, f"../output/pred/{self.run_name}-train.pkl")
# mlflow
self.run_id = mlflow.active_run().info.run_id
log_param("model_name", str(self.model_cls).split(".")[-1][:-2])
log_param("fe_name", self.fe_name)
log_param("train_params", self.params)
log_param("cv_strategy", str(self.cv))
log_param("evaluation_metric", self.evaluation_metric)
log_metric("cv_score", cv_score)
log_param(
"fold_scores",
dict(
zip(
[f"fold_{i}" for i in range(len(scores))],
[round(s, 4) for s in scores],
)),
)
log_param("cols_definition", self.cols_definition)
log_param("description", self.description)
mlflow.end_run()
test_dataset = MeliChallengeDataset(
dataset_path=test_data,
random_buffer_size=1
)
test_loader = DataLoader(
test_dataset,
batch_size=batch_size,
shuffle=False,
collate_fn=pad_sequences,
drop_last=False
)
else:
test_dataset = None
test_loader = None
mlflow.set_experiment(f"diplodatos.{language}.{exp_name}")
with mlflow.start_run():
logging.info("Starting experiment")
# Log all relevent hyperparameters
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
logging.info("Building classifier")
model = CNNClassifier(
pretrained_embeddings_path=pretrained_embeddings,
token_to_index=token_to_index,
n_labels=train_dataset.n_labels,
dropout=dropout,
batch_size=batch_size,
vector_size=embeddings_size,
