def train_random_forest(ntrees): with kiwi.start_run(): rf = H2ORandomForestEstimator(ntrees=ntrees) train_cols = [n for n in wine.col_names if n != "quality"] rf.train(train_cols, "quality", training_frame=train, validation_frame=test) kiwi.log_param("ntrees", ntrees) kiwi.log_metric("rmse", rf.rmse()) kiwi.log_metric("r2", rf.r2()) kiwi.log_metric("mae", rf.mae()) kiwi.h2o.log_model(rf, "model")
def test_log_param(): with start_run() as active_run: run_id = active_run.info.run_id kiwi.log_param("name_1", "a") kiwi.log_param("name_2", "b") kiwi.log_param("nested/nested/name", 5) finished_run = tracking.MlflowClient().get_run(run_id) # Validate params assert finished_run.data.params == { "name_1": "a", "name_2": "b", "nested/nested/name": "5" }
l1_ratio = float(sys.argv[2]) if len(sys.argv) > 2 else 0.5 with kiwi.start_run(): lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42) lr.fit(train_x, train_y) predicted_qualities = lr.predict(test_x) (rmse, mae, r2) = eval_metrics(test_y, predicted_qualities) print("Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio)) print(" RMSE: %s" % rmse) print(" MAE: %s" % mae) print(" R2: %s" % r2) kiwi.log_param("alpha", alpha) kiwi.log_param("l1_ratio", l1_ratio) kiwi.log_metric("rmse", rmse) kiwi.log_metric("r2", r2) kiwi.log_metric("mae", mae) tracking_url_type_store = urlparse(kiwi.get_tracking_uri()).scheme # Model registry does not work with file store if tracking_url_type_store != "file": # Register the model # There are other ways to use the Model Registry, which depends on the use case, # please refer to the doc for more information: # https://mlflow.org/docs/latest/model-registry.html#api-workflow kiwi.sklearn.log_model(lr,
test_accuracy = 100.0 * correct / len(test_loader.dataset) print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(test_loader.dataset), test_accuracy)) step = (epoch + 1) * len(train_loader) log_scalar('test_loss', test_loss, step) log_scalar('test_accuracy', test_accuracy, step) def log_scalar(name, value, step): """Log a scalar value to both MLflow and TensorBoard""" writer.add_scalar(name, value, step) kiwi.log_metric(name, value) with kiwi.start_run(): # Log our parameters into mlflow for key, value in vars(args).items(): kiwi.log_param(key, value) # Create a SummaryWriter to write TensorBoard events locally output_dir = dirpath = tempfile.mkdtemp() writer = SummaryWriter(output_dir) print("Writing TensorBoard events locally to %s\n" % output_dir) # Perform the training for epoch in range(1, args.epochs + 1): train(epoch) test(epoch) # Upload the TensorBoard event logs as a run artifact print("Uploading TensorBoard events as a run artifact...") kiwi.log_artifacts(output_dir, artifact_path="events") print("\nLaunch TensorBoard with:\n\ntensorboard --logdir=%s" %
def objective(args): # start a run with kiwi.start_run(experiment_id=current_experiment_id): # register hyperparams for key, value in args.items(): kiwi.log_param(key, value) # Define model model = RankedNetworkCNNModule(args['learning_rate'], dataset.get_embeddings(), hidden_dim=args['hidden'], output_labels=2) # Train (obviously) trainer = pl.Trainer(max_epochs=15, logger=KiwiLogger()) trainer.fit(model, train_loader, val_loader) # Evaluation on held-out test-set with torch.no_grad(): model.eval() results = pd.DataFrame(columns=['labels', 'predictions']) for batch_idx, batch in enumerate(test_loader): y_hat = model(batch['a'], batch['b']) results: pd.DataFrame = results.append(pd.DataFrame({ 'labels': batch['label'].flatten(), 'predictions': y_hat.detach().argmax(axis=1) }), ignore_index=True) results.to_csv() # With a nice confusion matrix confusion_matrix(y_pred=results['predictions'].values, y_true=results['labels'].values, classes=[0, 1]) cm = ConfusionMatrix( actual_vector=results['labels'].values, predict_vector=results['predictions'].values) output_test_results = "cm.txt" cm.save_stat(output_test_results) output_test_predictions_file = "test_predictions.txt" np.savetxt(output_test_predictions_file, results['predictions'].values, delimiter=",") kiwi.log_metric(key="test_acc", value=cm.Overall_ACC) kiwi.log_metric(key="test_f1_micro", value=cm.F1_Micro) kiwi.log_metric(key="test_f1_macro", value=cm.F1_Macro) kiwi.log_metric(key="test_ci_pm", value=cm.CI95[1] - cm.Overall_ACC) kiwi.log_metric(key="test_ci_pm", value=cm.CI95[1] - cm.Overall_ACC) kiwi.log_artifact(output_test_predictions_file) kiwi.log_artifact(output_test_results + ".pycm") return cm.Overall_ACC
def log_params(parameters): for k, v in parameters.items(): kiwi.log_param(k, v)
'experiement': [rand()], 'run': [rand()], 'because': [rand()], 'we': [rand()], 'need': [rand()], 'to': [rand()], 'check': [rand()], 'how': [rand()], 'it': [rand()], 'handles': [rand()], } log_metrics(metrics) with kiwi.start_run(run_name='sort_child.py', nested=True): kiwi.log_metric('test_metric', 1) kiwi.log_param('test_param', 1) with kiwi.start_run(run_name='sort_child.py', nested=True): kiwi.log_metric('test_metric', 2) kiwi.log_param('test_param', 2) # Grandchildren with kiwi.start_run(run_name='parent'): with kiwi.start_run(run_name='child', nested=True): with kiwi.start_run(run_name='grandchild', nested=True): pass # Loop loop_1_run_id = None loop_2_run_id = None with kiwi.start_run(run_name='loop-1') as run_1:
def test_search_runs(): kiwi.set_experiment("exp-for-search") # Create a run and verify that the current active experiment is the one we just set logged_runs = {} with kiwi.start_run() as active_run: logged_runs["first"] = active_run.info.run_id kiwi.log_metric("m1", 0.001) kiwi.log_metric("m2", 0.002) kiwi.log_metric("m1", 0.002) kiwi.log_param("p1", "a") kiwi.set_tag("t1", "first-tag-val") with kiwi.start_run() as active_run: logged_runs["second"] = active_run.info.run_id kiwi.log_metric("m1", 0.008) kiwi.log_param("p2", "aa") kiwi.set_tag("t2", "second-tag-val") def verify_runs(runs, expected_set): assert set([r.info.run_id for r in runs ]) == set([logged_runs[r] for r in expected_set]) experiment_id = MlflowClient().get_experiment_by_name( "exp-for-search").experiment_id # 2 runs in this experiment assert len(MlflowClient().list_run_infos(experiment_id, ViewType.ACTIVE_ONLY)) == 2 # 2 runs that have metric "m1" > 0.001 runs = MlflowClient().search_runs([experiment_id], "metrics.m1 > 0.0001") verify_runs(runs, ["first", "second"]) # 1 run with has metric "m1" > 0.002 runs = MlflowClient().search_runs([experiment_id], "metrics.m1 > 0.002") verify_runs(runs, ["second"]) # no runs with metric "m1" > 0.1 runs = MlflowClient().search_runs([experiment_id], "metrics.m1 > 0.1") verify_runs(runs, []) # 1 run with metric "m2" > 0 runs = MlflowClient().search_runs([experiment_id], "metrics.m2 > 0") verify_runs(runs, ["first"]) # 1 run each with param "p1" and "p2" runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'", ViewType.ALL) verify_runs(runs, ["first"]) runs = MlflowClient().search_runs([experiment_id], "params.p2 != 'a'", ViewType.ALL) verify_runs(runs, ["second"]) runs = MlflowClient().search_runs([experiment_id], "params.p2 = 'aa'", ViewType.ALL) verify_runs(runs, ["second"]) # 1 run each with tag "t1" and "t2" runs = MlflowClient().search_runs([experiment_id], "tags.t1 = 'first-tag-val'", ViewType.ALL) verify_runs(runs, ["first"]) runs = MlflowClient().search_runs([experiment_id], "tags.t2 != 'qwerty'", ViewType.ALL) verify_runs(runs, ["second"]) runs = MlflowClient().search_runs([experiment_id], "tags.t2 = 'second-tag-val'", ViewType.ALL) verify_runs(runs, ["second"]) # delete "first" run MlflowClient().delete_run(logged_runs["first"]) runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'", ViewType.ALL) verify_runs(runs, ["first"]) runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'", ViewType.DELETED_ONLY) verify_runs(runs, ["first"]) runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'", ViewType.ACTIVE_ONLY) verify_runs(runs, [])
import os import shutil import sys import random import tempfile import kiwi from kiwi import log_metric, log_param, log_artifacts, get_artifact_uri, active_run,\ get_tracking_uri, log_artifact if __name__ == "__main__": print("Running {} with tracking URI {}".format(sys.argv[0], get_tracking_uri())) log_param("param1", 5) log_metric("foo", 5) log_metric("foo", 6) log_metric("foo", 7) log_metric("random_int", random.randint(0, 100)) run_id = active_run().info.run_id # Get run metadata & data from the tracking server service = kiwi.tracking.MlflowClient() run = service.get_run(run_id) print("Metadata & data for run with UUID %s: %s" % (run_id, run)) local_dir = tempfile.mkdtemp() message = "test artifact written during run %s within artifact URI %s\n" \ % (active_run().info.run_id, get_artifact_uri()) try: file_path = os.path.join(local_dir, "some_output_file.txt") with open(file_path, "w") as handle: handle.write(message) log_artifacts(local_dir, "some_subdir")
def log_hyperparams(self, params): for key, value in vars(params).items(): kiwi.log_param(key, value)
import os from random import random, randint from kiwi import log_metric, log_param, log_artifacts if __name__ == "__main__": print("Running mlflow_tracking.py") log_param("param1", randint(0, 100)) log_metric("foo", random()) log_metric("foo", random() + 1) log_metric("foo", random() + 2) if not os.path.exists("outputs"): os.makedirs("outputs") with open("outputs/test.txt", "w") as f: f.write("hello world!") log_artifacts("outputs")
def train(image_files, labels, domain, image_width=224, image_height=224, epochs=1, batch_size=16, test_ratio=0.2, seed=None): """ Train VGG16 model on provided image files. This will create a new MLflow run and log all parameters, metrics and the resulting model with MLflow. The resulting model is an instance of KerasImageClassifierPyfunc - a custom python function model that embeds all necessary preprocessing together with the VGG16 Keras model. The resulting model can be applied directly to image base64 encoded image data. :param image_height: Height of the input image in pixels. :param image_width: Width of the input image in pixels. :param image_files: List of image files to be used for training. :param labels: List of labels for the image files. :param domain: Dictionary representing the domain of the reponse. Provides mapping label-name -> label-id. :param epochs: Number of epochs to train the model for. :param batch_size: Batch size used during training. :param test_ratio: Fraction of dataset to be used for validation. This data will not be used during training. :param seed: Random seed. Used e.g. when splitting the dataset into train / validation. """ assert len(set(labels)) == len(domain) input_shape = (image_width, image_height, 3) with kiwi.start_run() as run: kiwi.log_param("epochs", str(epochs)) kiwi.log_param("batch_size", str(batch_size)) kiwi.log_param("validation_ratio", str(test_ratio)) if seed: kiwi.log_param("seed", str(seed)) def _read_image(filename): with open(filename, "rb") as f: return f.read() with tf.Graph().as_default() as g: with tf.Session(graph=g).as_default(): dims = input_shape[:2] x = np.array([ decode_and_resize_image(_read_image(x), dims) for x in image_files ]) y = np_utils.to_categorical(np.array(labels), num_classes=len(domain)) train_size = 1 - test_ratio x_train, x_valid, y_train, y_valid = train_test_split( x, y, random_state=seed, train_size=train_size) model = _create_model(input_shape=input_shape, classes=len(domain)) model.compile(optimizer=keras.optimizers.SGD(decay=1e-5, nesterov=True, momentum=.9), loss=keras.losses.categorical_crossentropy, metrics=["accuracy"]) sorted_domain = sorted(domain.keys(), key=lambda x: domain[x]) model.fit(x=x_train, y=y_train, validation_data=(x_valid, y_valid), epochs=epochs, batch_size=batch_size, callbacks=[ MLflowLogger(model=model, x_train=x_train, y_train=y_train, x_valid=x_valid, y_valid=y_valid, artifact_path="model", domain=sorted_domain, image_dims=input_shape) ])