def _log_plotly_figure(): df = px.data.tips() plotly_fig = px.histogram(df, x="total_bill", y="tip", color="sex", marginal="rug", hover_data=df.columns) # Log plotly figure as interactive chart in the experiment' artifacts tab. log_chart(name='plotly_figure', chart=plotly_fig)
def log_img_and_chart( name: str, fig: matplotlib.figure.Figure, experiment: Optional[neptune.experiments.Experiment] = None, verbose: bool = False): if verbose: print("inside log_img_and_chart -> " + name) _exp = experiment if experiment else neptune log_chart(name, fig, _exp) _exp.log_image(name, fig) if verbose: print("leaving log_img_and_chart -> " + name)
def __call__(self, study, trial): import optuna.visualization as vis self.exp.log_metric('run_score', trial.value) self.exp.log_metric('best_so_far_run_score', study.best_value) self.exp.log_text('run_parameters', str(trial.params)) if self.log_study: pickle_and_log_artifact(study, 'study.pkl', experiment=self.exp) if self.log_optimization_history: log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=self.exp) if self.log_contour: log_chart(name='contour', chart=vis.plot_contour(study, params=self.params), experiment=self.exp) if self.log_parallel_coordinate: log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=self.params), experiment=self.exp) if self.log_slice: log_chart(name='slice', chart=vis.plot_slice(study, params=self.params), experiment=self.exp)
def _log_bokeh_figure(): palette2 = tuple(reversed(pltt)) counties = { code: county for code, county in cnts.items() if county["state"] == "tx" } county_xs = [county["lons"] for county in counties.values()] county_ys = [county["lats"] for county in counties.values()] county_names = [county['name'] for county in counties.values()] county_rates = [unemployment[county_id] for county_id in counties] color_mapper = LogColorMapper(palette=palette2) chart_data = dict( x=county_xs, y=county_ys, name=county_names, rate=county_rates, ) TOOLS = "pan,wheel_zoom,reset,hover,save" p = figure(title="Texas Unemployment, 2009", tools=TOOLS, x_axis_location=None, y_axis_location=None, tooltips=[("Name", "@name"), ("Unemployment rate", "@rate%"), ("(Long, Lat)", "($x, $y)")]) p.grid.grid_line_color = None p.hover.point_policy = "follow_mouse" p.patches('x', 'y', source=chart_data, fill_color={ 'field': 'rate', 'transform': color_mapper }, fill_alpha=0.7, line_color="white", line_width=0.5) # Log bokeh figure as interactive chart in the experiment' artifacts tab. log_chart(name='bokeh_figure', chart=p)
def log_study_info(study, experiment=None, log_charts=True, params=None): """Logs runs results and parameters to neptune. Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric), best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts ('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder. Args: study('optuna.study.Study'): Optuna study object after training is completed. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. log_charts('bool'): Whether optuna visualization charts should be logged. By default all charts are logged. params(`list`): List of parameters to be visualized. Default is all parameters. Examples: Initialize neptune_monitor:: import neptune import neptunecontrib.monitoring.optuna as opt_utils neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') neptune.create_experiment(name='optuna sweep') neptune_callback = opt_utils.NeptuneCallback() Run Optuna training passing monitor as callback:: ... study = optuna.create_study(direction='maximize') study.optimize(objective, n_trials=100, callbacks=[neptune_callback]) opt_utils.log_study_info(study) You can explore an example experiment in Neptune: https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts """ import optuna.visualization as vis _exp = experiment if experiment else neptune _exp.log_metric('best_score', study.best_value) _exp.set_property('best_parameters', study.best_params) if log_charts: log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=_exp) log_chart(name='contour', chart=vis.plot_contour(study, params=params), experiment=_exp) log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=params), experiment=_exp) log_chart(name='slice', chart=vis.plot_slice(study, params=params), experiment=_exp) pickle_and_log_artifact(study, 'study.pkl', experiment=_exp)
def _log_altair_figure(): source = data.cars() brush = alt.selection(type='interval') points = alt.Chart(source).mark_point().encode( x='Horsepower:Q', y='Miles_per_Gallon:Q', color=alt.condition(brush, 'Origin:N', alt.value('lightgray'))).add_selection(brush) bars = alt.Chart(source).mark_bar().encode( y='Origin:N', color='Origin:N', x='count(Origin):Q').transform_filter(brush) chart = points & bars # Log altair figure as interactive chart in the experiment' artifacts tab. log_chart(name='altair_chart', chart=chart)
def _log_matplotlib_figure(): def _scatter_hist(x, y, ax, ax_histx, ax_histy): ax_histx.tick_params(axis="x", labelbottom=False) ax_histy.tick_params(axis="y", labelleft=False) ax.scatter(x, y) binwidth = 0.25 xymax = max(np.max(np.abs(x)), np.max(np.abs(y))) lim = (int(xymax / binwidth) + 1) * binwidth bins = np.arange(-lim, lim + binwidth, binwidth) ax_histx.hist(x, bins=bins) ax_histy.hist(y, bins=bins, orientation='horizontal') np.random.seed(19680801) x = np.random.randn(1000) y = np.random.randn(1000) fig = plt.figure(figsize=(8, 8)) gs = fig.add_gridspec(2, 2, width_ratios=(7, 2), height_ratios=(2, 7), left=0.1, right=0.9, bottom=0.1, top=0.9, wspace=0.05, hspace=0.05) ax = fig.add_subplot(gs[1, 0]) ax_histx = fig.add_subplot(gs[0, 0], sharex=ax) ax_histy = fig.add_subplot(gs[1, 1], sharey=ax) _scatter_hist(x, y, ax, ax_histx, ax_histy) # Log matplotlib figure as interactive chart in the experiment' artifacts tab. log_chart(name='matplotlib_figure', chart=fig) plt.close('all')
def log_local_explanations(explainer, observation, experiment=None): """Logs local explanations from dalex to Neptune. Dalex explanations are converted to interactive HTML objects and then uploaded to Neptune as an artifact with path charts/{name}.html. The following explanations are logged: break down, break down with interactions, shap, ceteris paribus, and ceteris paribus for categorical variables. Explanation charts are created and logged with default settings. To log charts with custom settings, create a custom chart and use `neptunecontrib.api.log_chart`. For more information about Dalex go to `Dalex Website <https://modeloriented.github.io/DALEX/>`_. Args: explainer (:obj:`dalex.Explainer`): an instance of dalex explainer observation (:obj): an observation that can be fed to the classifier for which the explainer was created experiment (:obj:`neptune.experiments.Experiment`, optional, default is ``None``): | For advanced users only. Pass Neptune `Experiment <https://docs.neptune.ai/neptune-client/docs/experiment.html#neptune.experiments.Experiment>`_ object if you want to control to which experiment data is logged. | If ``None``, log to currently active, and most recent experiment. Examples: Start an experiment:: import neptune neptune.init(api_token='ANONYMOUS', project_qualified_name='shared/dalex-integration') neptune.create_experiment(name='logging explanations') Train your model and create dalex explainer:: ... clf.fit(X, y) expl = dx.Explainer(clf, X, y, label="Titanic MLP Pipeline") new_observation = pd.DataFrame({'gender': ['male'], 'age': [25], 'class': ['1st'], 'embarked': ['Southampton'], 'fare': [72], 'sibsp': [0], 'parch': 0}, index=['John']) log_local_explanations(expl, new_observation) Note: Check out how the logged explanations look in Neptune: `example experiment <https://ui.neptune.ai/o/shared/org/dalex-integration/e/DAL-48/artifacts?path=charts%2F>`_ """ _exp = experiment if experiment else neptune bd = explainer.predict_parts(observation, type='break_down') bd_interactions = explainer.predict_parts(observation, type='break_down_interactions') sh = explainer.predict_parts(observation, type='shap') cp = explainer.predict_profile(observation) bd_plot = bd.plot(show=False) log_chart(name='Break Down', chart=bd_plot, experiment=_exp) bd_interactions_plot = bd_interactions.plot(show=False) log_chart(name='Break Down Interactions', chart=bd_interactions_plot, experiment=_exp) sh_plot = sh.plot(show=False) log_chart(name='SHAP', chart=sh_plot, experiment=_exp) cp_plot = cp.plot(show=False) log_chart(name="Ceteris Paribus", chart=cp_plot, experiment=_exp) cp_plot_cat = cp.plot(variable_type="categorical", show=False) log_chart(name="Ceteris Paribus Categorical", chart=cp_plot_cat, experiment=_exp)
def log_global_explanations(explainer, categorical_features=None, numerical_features=None, experiment=None): """Logs global explanations from dalex to Neptune. Dalex explanations are converted to interactive HTML objects and then uploaded to Neptune as an artifact with path charts/{name}.html. The following explanations are logged: variable importance. If categorical features are specified partial dependence and accumulated dependence are also logged. Explanation charts are created and logged with default settings. To log charts with custom settings, create a custom chart and use `neptunecontrib.api.log_chart`. For more information about Dalex go to `Dalex Website <https://modeloriented.github.io/DALEX/>`_. Args: explainer (:obj:`dalex.Explainer`): an instance of dalex explainer categorical_features (:list): list of categorical features for which you want to create accumulated dependence plots. numerical_features (:list): list of numerical features for which you want to create partial dependence plots. experiment (:obj:`neptune.experiments.Experiment`, optional, default is ``None``): | For advanced users only. Pass Neptune `Experiment <https://docs.neptune.ai/neptune-client/docs/experiment.html#neptune.experiments.Experiment>`_ object if you want to control to which experiment data is logged. | If ``None``, log to currently active, and most recent experiment. Examples: Start an experiment:: import neptune neptune.init(api_token='ANONYMOUS', project_qualified_name='shared/dalex-integration') neptune.create_experiment(name='logging explanations') Train your model and create dalex explainer:: ... clf.fit(X, y) expl = dx.Explainer(clf, X, y, label="Titanic MLP Pipeline") log_global_explanations(expl, categorical_features=["gender", "class"], numerical_features=["age", "fare"]) Note: Check out how the logged explanations look in Neptune: `example experiment <https://ui.neptune.ai/o/shared/org/dalex-integration/e/DAL-48/artifacts?path=charts%2F>`_ """ _exp = experiment if experiment else neptune vi = explainer.model_parts() vi_plot = vi.plot(show=False) log_chart(name="Variable Importance", chart=vi_plot, experiment=_exp) if numerical_features: pdp_num = explainer.model_profile(type='partial', variables=numerical_features) pdp_num.result["_label_"] = 'pdp' ale_num = explainer.model_profile(type='accumulated', variables=numerical_features) ale_num.result["_label_"] = 'ale' pdp_num_plot = pdp_num.plot(ale_num, show=False) log_chart(name="Aggregated Profiles Numerical", chart=pdp_num_plot, experiment=_exp) if categorical_features: pdp_cat = explainer.model_profile(type='partial', variable_type='categorical', variables=categorical_features) pdp_cat.result['_label_'] = 'pdp' ale_cat = explainer.model_profile(type='accumulated', variable_type='categorical', variables=categorical_features) ale_cat.result['_label_'] = 'ale' ale_cat_plot = ale_cat.plot(pdp_cat, show=False) log_chart(name="Aggregated Profiles Categorical", chart=ale_cat_plot, experiment=_exp)
## Create chart import matplotlib.pyplot as plt from scikitplot.metrics import plot_roc y_test_pred = model.predict(x_test) fig, ax = plt.subplots() plot_roc(y_test, y_test_pred, ax=ax) ## Log chart to Neptune as interactive Plotly chart from neptunecontrib.api import log_chart log_chart(name='ROC curve', chart=fig) # tests exp = neptune.get_experiment() neptune.stop() # tests all_logs = exp.get_logs() ## check logs correct_logs = ['loss', 'accuracy', 'val_loss', 'val_accuracy'] if set(all_logs.keys()) != set(correct_logs):
def test_main(args, neptune): # some constants error_scaler = 1E8 ar = [1240, 1460] # anomaly range @200608-03:10 in_n = args.dim_input # load model and obtain some stats model = torch.load(args.out_dir + '/' + args.exp_id).to('cpu') fp = open(args.stat_file, 'r') lines = fp.readlines() x_avg = torch.tensor([float(s) for s in lines[0].split(',')]) x_std = torch.tensor([float(s) for s in lines[1].split(',')]) fp.close() # 1. load test data val_data = np.loadtxt(args.val_path, delimiter=',') val_data = torch.tensor(val_data).type(torch.float32) val_recon = forward(val_data, model, x_avg, x_std, args) val_err = torch.sum((val_recon - val_data)**2, dim=1, keepdim=True) # squared error ve = val_err * error_scaler test_data = np.loadtxt(args.test_path, delimiter=',') test_lbl = test_data[:, -1] data_len = test_data.shape[0] # retreive labels and length test_data = torch.tensor(test_data[:, :-1]).type(torch.float32) test_recon = forward(test_data, model, x_avg, x_std, args) test_err = torch.sum((test_recon - test_data)**2, dim=1, keepdim=True) # squared error te = test_err * error_scaler # 2. measure validation error and test error neptune.set_property('validation error', torch.sum(ve).item()) neptune.set_property('test error', torch.sum(te).item()) # 2. plot reconstruction results cols = ['sensor1', 'sensor2'] # features ids_col = range(test_data.shape[0]) # for index for j, (data, recon, str) in enumerate([(val_data, val_recon, 'Validation'), (test_data, test_recon, 'Test')]): fig, axs = plt.subplots(len(cols), 1, figsize=(12, 3)) for i, col in enumerate(cols): axs[i].plot(ids_col, data.numpy()[:, i], '-c', linewidth=2, label='Raw Data') axs[i].plot(ids_col, recon.detach().numpy()[:, i], '-b', linewidth=1, label='Reconstructed Data') axs[1].legend() # only add legend for second row fig.suptitle('Time Series of ' + str) log_chart('Data-Reconstruction', fig) #3. find threshold T = (torch.mean(ve) + 2 * torch.std(ve)).item() T_ = np.empty((data_len, 1)) T_[:] = T # for plotting threshold if args.use_smoothing == 1: ve = smooth(ve.detach().numpy(), args.window_size) te = smooth( te.detach().numpy(), args.window_size ) # smoothing removes first window_size samples. (Tx, 1) -> (Tx-args.window_size+1, 1) pred = (te > T) # pred is classification result pad = np.empty((args.window_size - 1, 1)) pad[:] = 0 pred = np.vstack([pad, pred]) # add 0 padding from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score acc = accuracy_score(test_lbl, pred) neptune.set_property('acc', acc) prec = precision_score(test_lbl, pred) neptune.set_property('prec', prec) rec = recall_score(test_lbl, pred) neptune.set_property('rec', rec) f1 = f1_score(test_lbl, pred) neptune.set_property('f1', f1) # 4. draw plot fig = plt.figure(figsize=(24, 4)) ids = list(range(data_len)) pad[:] = np.nan te = np.vstack([pad, te]) # add nan padding plt.plot(ids[:ar[0]], te[:ar[0]], '-c', label='Test Reconstruction Error (Normal)') plt.plot(ids[ar[0]:ar[1]], te[ar[0]:ar[1]], '-r', label='Test Reconstruction Error (Anomaly)') plt.plot(ids[ar[1]:], te[ar[1]:], '-c') plt.plot(ids, T_, '--b', label='Threshold') plt.xlabel('Time') plt.ylabel('Error') plt.legend() #plt.ylim((0,2E5)) plt.title('Reconstruction Error') log_chart('Reconstruction Error', fig)
def train(train_dataloader, eval_dataloader, model, config, neptune_project, freeze_model=False): """ :param train_dataset: iterator on the training set :param eval_dataset: iterator on the test set :param model: instance of the model to train :param processor: processor object used for evaluation :param config: Config :param freeze_model: whether or not to freeze BERT """ if config["resume_training"]: # retrieving and updating already existing experiment exp = neptune_project.get_experiments(id=config["neptune_id"])[0] else: # creating a neptune experiment exp = neptune_project.create_experiment(name="{}_{}".format(config["model_type"], str(datetime.now())), params=config, upload_source_files=['*.py', "models/", "utils/"], tags=[config["model_type"]] + config["tags"]) num_training_steps = len(train_dataloader) * config['num_train_epochs'] // config['gradient_accumulation_steps'] no_decay = ['bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': config['weight_decay']}, {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] warmup_steps = math.ceil(num_training_steps * config['warmup_ratio']) config['warmup_steps'] = warmup_steps if config['warmup_steps'] else 0 optimizer = AdamW(optimizer_grouped_parameters, lr=config['learning_rate'], eps=config['adam_epsilon']) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=config['warmup_steps'], num_training_steps=num_training_steps) # will freeze all the model parameters except the classification part if freeze_model: model.freeze_bert_encoder() # optimization if config['fp16']: try: from apex import amp except ImportError: raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") model, optimizer = amp.initialize(model, optimizer, opt_level=config['fp16_opt_level']) # if running on multiple GPUs if config['n_gpu'] > 1: model = torch.nn.DataParallel(model) logging.info("***** Running training *****") logging.info(" Num examples = %d", len(train_dataloader)) logging.info(" Num Epochs = %d", config['num_train_epochs']) logging.info(" Total train batch size = %d", config['train_batch_size']) logging.info(" Gradient Accumulation steps = %d", config['gradient_accumulation_steps']) logging.info(" Total optimization steps = %d", num_training_steps) tr_loss, train_acc, logging_loss = 0.0, 0.0, 0.0 model.zero_grad() start_epoch = int(config.get("previous_checkpoint", 0) / len(train_dataloader)) global_step = start_epoch * len(train_dataloader) train_iterator = trange(start_epoch, int(config['num_train_epochs']), desc="Epoch") # starting training for epoch in train_iterator: epoch_losses = [] for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")): # avoiding to feed already seen batches when resuming training if config["resume_training"] and global_step < config.get("previous_checkpoint") + 1: global_step += 1 continue model.train() if 'distilbert' not in config['model_type']: inputs = {'input_ids': batch["input_ids"].to(device), 'attention_mask': batch["input_mask"].to(device), 'token_type_ids': batch["token_type_ids"].to(device) if config['model_type'] in ['bert', 'xlnet'] else None, 'labels': batch["labels"]} else: inputs = {'input_ids': batch["input_ids"].to(device), 'attention_mask': batch["input_mask"].to(device), 'labels': batch["labels"].to(device)} outputs = model(**inputs) loss, logits = outputs[:2] # model outputs are always tuple in pytorch-transformers (see doc) # handle multi-gpus run if config["n_gpu"] > 1: loss = loss.mean() print("\r%f" % loss, end='') epoch_losses.append(loss.item()) if config['task_name'] == "multi-label": with torch.no_grad(): logits = logits.sigmoid() train_acc += accuracy_thresh(logits, inputs["labels"]) else: train_acc += get_accuracy(logits.detach().cpu().numpy(), batch[3].detach().cpu().numpy()) # gradient accumulation if config['gradient_accumulation_steps'] > 1: loss = loss / config['gradient_accumulation_steps'] # optimization if config['fp16']: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), config['max_grad_norm']) else: loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), config['max_grad_norm']) tr_loss += loss.item() if (step + 1) % config['gradient_accumulation_steps'] == 0: optimizer.step() scheduler.step() # Update learning rate schedule model.zero_grad() global_step += 1 if config['logging_steps'] > 0 and global_step % config['logging_steps'] == 0: exp.log_metric(log_name='lr', y=scheduler.get_lr()[0], x=global_step) exp.log_metric(log_name='train_loss', y=(tr_loss - logging_loss) / config['logging_steps'], x=global_step) exp.log_metric(log_name='train_acc', y=train_acc / config['logging_steps'], x=global_step) logging_loss = tr_loss train_acc = 0.0 if config['save_steps'] > 0 and global_step % config['save_steps'] == 0: # Save model checkpoint output_dir = os.path.join(config['output_dir'], 'checkpoint-{}'.format(global_step)) if not os.path.exists(output_dir): os.makedirs(output_dir) model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training model_to_save.save_pretrained(output_dir) logging.info("Saving model checkpoint to %s", output_dir) exp.log_artifact(os.path.join(output_dir, "pytorch_model.bin"), "pytorch_model_{}.bin".format(global_step)) # Log metrics if config['evaluate_during_training']: results = evaluate(eval_dataloader, model, config, epoch) for key, value in results["scalars"].items(): exp.log_metric(log_name='eval_{}'.format(key), y=value, x=epoch) if "labels_probs" in results["arrays"].keys(): labels_probs = results["arrays"]["labels_probs"] for i in range(labels_probs.shape[0]): fig = plt.figure(figsize=(15, 15)) sns.distplot(labels_probs[i], kde=False, bins=100) plt.title("Probability boxplot for label {}".format(i)) log_chart(name="dist_label_{}_epoch_{}".format(i, epoch), chart=fig, experiment=exp) plt.close("all") return global_step, tr_loss / global_step
def log_study_info(study, experiment=None, log_study=True, log_charts=True, log_optimization_history=False, log_contour=False, log_parallel_coordinate=False, log_slice=False, params=None): """Logs runs results and parameters to neptune. Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric), best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts ('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder. Args: study('optuna.study.Study'): Optuna study object after training is completed. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. log_study('bool'): Whether optuna study object should be logged as pickle. Default is True. log_charts('bool'): Deprecated argument. Whether all optuna visualizations charts should be logged. By default all charts are sent. To not log any charts set log_charts=False. If you want to log a particular chart change the argument for that chart explicitly. For example log_charts=False and log_slice=True will log only the slice plot to Neptune. log_optimization_history('bool'): Whether optuna optimization history chart should be logged. Default is True. log_contour('bool'): Whether optuna contour plot should be logged. Default is True. log_parallel_coordinate('bool'): Whether optuna parallel coordinate plot should be logged. Default is True. log_slice('bool'): Whether optuna slice chart should be logged. Default is True. params(`list`): List of parameters to be visualized. Default is all parameters. Examples: Initialize neptune_monitor:: import neptune import neptunecontrib.monitoring.optuna as opt_utils neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') neptune.create_experiment(name='optuna sweep') neptune_callback = opt_utils.NeptuneCallback() Run Optuna training passing monitor as callback:: ... study = optuna.create_study(direction='maximize') study.optimize(objective, n_trials=100, callbacks=[neptune_callback]) opt_utils.log_study_info(study) You can explore an example experiment in Neptune: https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts """ import optuna.visualization as vis _exp = experiment if experiment else neptune _exp.log_metric('best_score', study.best_value) _exp.set_property('best_parameters', study.best_params) if log_charts: message = """log_charts argument is depraceted and will be removed in future releases. Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly. """ warnings.warn(message) log_optimization_history = True log_contour = True log_parallel_coordinate = True log_slice = True if log_study: pickle_and_log_artifact(study, 'study.pkl', experiment=_exp) if log_optimization_history: log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=_exp) if log_contour: log_chart(name='contour', chart=vis.plot_contour(study, params=params), experiment=_exp) if log_parallel_coordinate: log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=params), experiment=_exp) if log_slice: log_chart(name='slice', chart=vis.plot_slice(study, params=params), experiment=_exp)
# Log model weights model.save('model') neptune.log_artifact('model') # Evaluate model eval_metrics = model.evaluate(x_test, y_test, verbose=0) for j, metric in enumerate(eval_metrics): neptune.log_metric('test_{}'.format(model.metrics_names[j]), metric) # Log predictions as table y_pred_proba = model.predict(x_test) y_pred = np.argmax(y_pred_proba, axis=1) y_pred = y_pred df = pd.DataFrame( data={ 'y_test': y_test, 'y_pred': y_pred, 'y_pred_probability': y_pred_proba.max(axis=1) }) log_table('predictions', df) # Log model performance visualizations fig, ax = plt.subplots() plot_roc(y_test, y_pred_proba, ax=ax) log_chart('ROC', fig) fig, ax = plt.subplots() plot_precision_recall(y_test, y_pred_proba, ax=ax) log_chart('precision recall', fig) plt.close('all')