def _log_plotly_figure():
df = px.data.tips()
plotly_fig = px.histogram(df,
x="total_bill",
y="tip",
color="sex",
marginal="rug",
hover_data=df.columns)
# Log plotly figure as interactive chart in the experiment' artifacts tab.
log_chart(name='plotly_figure', chart=plotly_fig)
def log_img_and_chart(
name: str,
fig: matplotlib.figure.Figure,
experiment: Optional[neptune.experiments.Experiment] = None,
verbose: bool = False):
if verbose:
print("inside log_img_and_chart -> " + name)
_exp = experiment if experiment else neptune
log_chart(name, fig, _exp)
_exp.log_image(name, fig)
if verbose:
print("leaving log_img_and_chart -> " + name)
def __call__(self, study, trial):
import optuna.visualization as vis
self.exp.log_metric('run_score', trial.value)
self.exp.log_metric('best_so_far_run_score', study.best_value)
self.exp.log_text('run_parameters', str(trial.params))
if self.log_study:
pickle_and_log_artifact(study, 'study.pkl', experiment=self.exp)
if self.log_optimization_history:
log_chart(name='optimization_history',
chart=vis.plot_optimization_history(study),
experiment=self.exp)
if self.log_contour:
log_chart(name='contour',
chart=vis.plot_contour(study, params=self.params),
experiment=self.exp)
if self.log_parallel_coordinate:
log_chart(name='parallel_coordinate',
chart=vis.plot_parallel_coordinate(study,
params=self.params),
experiment=self.exp)
if self.log_slice:
log_chart(name='slice',
chart=vis.plot_slice(study, params=self.params),
experiment=self.exp)
def _log_bokeh_figure():
palette2 = tuple(reversed(pltt))
counties = {
code: county
for code, county in cnts.items() if county["state"] == "tx"
}
county_xs = [county["lons"] for county in counties.values()]
county_ys = [county["lats"] for county in counties.values()]
county_names = [county['name'] for county in counties.values()]
county_rates = [unemployment[county_id] for county_id in counties]
color_mapper = LogColorMapper(palette=palette2)
chart_data = dict(
x=county_xs,
y=county_ys,
name=county_names,
rate=county_rates,
)
TOOLS = "pan,wheel_zoom,reset,hover,save"
p = figure(title="Texas Unemployment, 2009",
tools=TOOLS,
x_axis_location=None,
y_axis_location=None,
tooltips=[("Name", "@name"), ("Unemployment rate", "@rate%"),
("(Long, Lat)", "($x, $y)")])
p.grid.grid_line_color = None
p.hover.point_policy = "follow_mouse"
p.patches('x',
'y',
source=chart_data,
fill_color={
'field': 'rate',
'transform': color_mapper
},
fill_alpha=0.7,
line_color="white",
line_width=0.5)
# Log bokeh figure as interactive chart in the experiment' artifacts tab.
log_chart(name='bokeh_figure', chart=p)
def log_study_info(study, experiment=None, log_charts=True, params=None):
"""Logs runs results and parameters to neptune.
Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric),
best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts
('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder.
Args:
study('optuna.study.Study'): Optuna study object after training is completed.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
log_charts('bool'): Whether optuna visualization charts should be logged. By default all charts are logged.
params(`list`): List of parameters to be visualized. Default is all parameters.
Examples:
Initialize neptune_monitor::
import neptune
import neptunecontrib.monitoring.optuna as opt_utils
neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME')
neptune.create_experiment(name='optuna sweep')
neptune_callback = opt_utils.NeptuneCallback()
Run Optuna training passing monitor as callback::
...
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=100, callbacks=[neptune_callback])
opt_utils.log_study_info(study)
You can explore an example experiment in Neptune:
https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts
"""
import optuna.visualization as vis
_exp = experiment if experiment else neptune
_exp.log_metric('best_score', study.best_value)
_exp.set_property('best_parameters', study.best_params)
if log_charts:
log_chart(name='optimization_history',
chart=vis.plot_optimization_history(study),
experiment=_exp)
log_chart(name='contour',
chart=vis.plot_contour(study, params=params),
experiment=_exp)
log_chart(name='parallel_coordinate',
chart=vis.plot_parallel_coordinate(study, params=params),
experiment=_exp)
log_chart(name='slice',
chart=vis.plot_slice(study, params=params),
experiment=_exp)
pickle_and_log_artifact(study, 'study.pkl', experiment=_exp)
def _log_altair_figure():
source = data.cars()
brush = alt.selection(type='interval')
points = alt.Chart(source).mark_point().encode(
x='Horsepower:Q',
y='Miles_per_Gallon:Q',
color=alt.condition(brush, 'Origin:N',
alt.value('lightgray'))).add_selection(brush)
bars = alt.Chart(source).mark_bar().encode(
y='Origin:N', color='Origin:N',
x='count(Origin):Q').transform_filter(brush)
chart = points & bars
# Log altair figure as interactive chart in the experiment' artifacts tab.
log_chart(name='altair_chart', chart=chart)
def _log_matplotlib_figure():
def _scatter_hist(x, y, ax, ax_histx, ax_histy):
ax_histx.tick_params(axis="x", labelbottom=False)
ax_histy.tick_params(axis="y", labelleft=False)
ax.scatter(x, y)
binwidth = 0.25
xymax = max(np.max(np.abs(x)), np.max(np.abs(y)))
lim = (int(xymax / binwidth) + 1) * binwidth
bins = np.arange(-lim, lim + binwidth, binwidth)
ax_histx.hist(x, bins=bins)
ax_histy.hist(y, bins=bins, orientation='horizontal')
np.random.seed(19680801)
x = np.random.randn(1000)
y = np.random.randn(1000)
fig = plt.figure(figsize=(8, 8))
gs = fig.add_gridspec(2,
2,
width_ratios=(7, 2),
height_ratios=(2, 7),
left=0.1,
right=0.9,
bottom=0.1,
top=0.9,
wspace=0.05,
hspace=0.05)
ax = fig.add_subplot(gs[1, 0])
ax_histx = fig.add_subplot(gs[0, 0], sharex=ax)
ax_histy = fig.add_subplot(gs[1, 1], sharey=ax)
_scatter_hist(x, y, ax, ax_histx, ax_histy)
# Log matplotlib figure as interactive chart in the experiment' artifacts tab.
log_chart(name='matplotlib_figure', chart=fig)
plt.close('all')
def log_local_explanations(explainer, observation, experiment=None):
"""Logs local explanations from dalex to Neptune.
Dalex explanations are converted to interactive HTML objects and then uploaded to Neptune
as an artifact with path charts/{name}.html.
The following explanations are logged: break down, break down with interactions, shap, ceteris paribus,
and ceteris paribus for categorical variables. Explanation charts are created and logged with default settings.
To log charts with custom settings, create a custom chart and use `neptunecontrib.api.log_chart`.
For more information about Dalex go to `Dalex Website <https://modeloriented.github.io/DALEX/>`_.
Args:
explainer (:obj:`dalex.Explainer`): an instance of dalex explainer
observation (:obj): an observation that can be fed to the classifier for which the explainer was created
experiment (:obj:`neptune.experiments.Experiment`, optional, default is ``None``):
| For advanced users only. Pass Neptune
`Experiment <https://docs.neptune.ai/neptune-client/docs/experiment.html#neptune.experiments.Experiment>`_
object if you want to control to which experiment data is logged.
| If ``None``, log to currently active, and most recent experiment.
Examples:
Start an experiment::
import neptune
neptune.init(api_token='ANONYMOUS',
project_qualified_name='shared/dalex-integration')
neptune.create_experiment(name='logging explanations')
Train your model and create dalex explainer::
...
clf.fit(X, y)
expl = dx.Explainer(clf, X, y, label="Titanic MLP Pipeline")
new_observation = pd.DataFrame({'gender': ['male'],
'age': [25],
'class': ['1st'],
'embarked': ['Southampton'],
'fare': [72],
'sibsp': [0],
'parch': 0},
index=['John'])
log_local_explanations(expl, new_observation)
Note:
Check out how the logged explanations look in Neptune:
`example experiment <https://ui.neptune.ai/o/shared/org/dalex-integration/e/DAL-48/artifacts?path=charts%2F>`_
"""
_exp = experiment if experiment else neptune
bd = explainer.predict_parts(observation, type='break_down')
bd_interactions = explainer.predict_parts(observation,
type='break_down_interactions')
sh = explainer.predict_parts(observation, type='shap')
cp = explainer.predict_profile(observation)
bd_plot = bd.plot(show=False)
log_chart(name='Break Down', chart=bd_plot, experiment=_exp)
bd_interactions_plot = bd_interactions.plot(show=False)
log_chart(name='Break Down Interactions',
chart=bd_interactions_plot,
experiment=_exp)
sh_plot = sh.plot(show=False)
log_chart(name='SHAP', chart=sh_plot, experiment=_exp)
cp_plot = cp.plot(show=False)
log_chart(name="Ceteris Paribus", chart=cp_plot, experiment=_exp)
cp_plot_cat = cp.plot(variable_type="categorical", show=False)
log_chart(name="Ceteris Paribus Categorical",
chart=cp_plot_cat,
experiment=_exp)
def log_global_explanations(explainer,
categorical_features=None,
numerical_features=None,
experiment=None):
"""Logs global explanations from dalex to Neptune.
Dalex explanations are converted to interactive HTML objects and then uploaded to Neptune
as an artifact with path charts/{name}.html.
The following explanations are logged: variable importance. If categorical features are specified partial dependence
and accumulated dependence are also logged. Explanation charts are created and logged with default settings.
To log charts with custom settings, create a custom chart and use `neptunecontrib.api.log_chart`.
For more information about Dalex go to `Dalex Website <https://modeloriented.github.io/DALEX/>`_.
Args:
explainer (:obj:`dalex.Explainer`): an instance of dalex explainer
categorical_features (:list): list of categorical features for which you want to create
accumulated dependence plots.
numerical_features (:list): list of numerical features for which you want to create
partial dependence plots.
experiment (:obj:`neptune.experiments.Experiment`, optional, default is ``None``):
| For advanced users only. Pass Neptune
`Experiment <https://docs.neptune.ai/neptune-client/docs/experiment.html#neptune.experiments.Experiment>`_
object if you want to control to which experiment data is logged.
| If ``None``, log to currently active, and most recent experiment.
Examples:
Start an experiment::
import neptune
neptune.init(api_token='ANONYMOUS',
project_qualified_name='shared/dalex-integration')
neptune.create_experiment(name='logging explanations')
Train your model and create dalex explainer::
...
clf.fit(X, y)
expl = dx.Explainer(clf, X, y, label="Titanic MLP Pipeline")
log_global_explanations(expl, categorical_features=["gender", "class"], numerical_features=["age", "fare"])
Note:
Check out how the logged explanations look in Neptune:
`example experiment <https://ui.neptune.ai/o/shared/org/dalex-integration/e/DAL-48/artifacts?path=charts%2F>`_
"""
_exp = experiment if experiment else neptune
vi = explainer.model_parts()
vi_plot = vi.plot(show=False)
log_chart(name="Variable Importance", chart=vi_plot, experiment=_exp)
if numerical_features:
pdp_num = explainer.model_profile(type='partial',
variables=numerical_features)
pdp_num.result["_label_"] = 'pdp'
ale_num = explainer.model_profile(type='accumulated',
variables=numerical_features)
ale_num.result["_label_"] = 'ale'
pdp_num_plot = pdp_num.plot(ale_num, show=False)
log_chart(name="Aggregated Profiles Numerical",
chart=pdp_num_plot,
experiment=_exp)
if categorical_features:
pdp_cat = explainer.model_profile(type='partial',
variable_type='categorical',
variables=categorical_features)
pdp_cat.result['_label_'] = 'pdp'
ale_cat = explainer.model_profile(type='accumulated',
variable_type='categorical',
variables=categorical_features)
ale_cat.result['_label_'] = 'ale'
ale_cat_plot = ale_cat.plot(pdp_cat, show=False)
log_chart(name="Aggregated Profiles Categorical",
chart=ale_cat_plot,
experiment=_exp)
## Create chart import matplotlib.pyplot as plt from scikitplot.metrics import plot_roc y_test_pred = model.predict(x_test) fig, ax = plt.subplots() plot_roc(y_test, y_test_pred, ax=ax) ## Log chart to Neptune as interactive Plotly chart from neptunecontrib.api import log_chart log_chart(name='ROC curve', chart=fig) # tests exp = neptune.get_experiment() neptune.stop() # tests all_logs = exp.get_logs() ## check logs correct_logs = ['loss', 'accuracy', 'val_loss', 'val_accuracy'] if set(all_logs.keys()) != set(correct_logs):
def test_main(args, neptune):
# some constants
error_scaler = 1E8
ar = [1240, 1460] # anomaly range @200608-03:10
in_n = args.dim_input
# load model and obtain some stats
model = torch.load(args.out_dir + '/' + args.exp_id).to('cpu')
fp = open(args.stat_file, 'r')
lines = fp.readlines()
x_avg = torch.tensor([float(s) for s in lines[0].split(',')])
x_std = torch.tensor([float(s) for s in lines[1].split(',')])
fp.close()
# 1. load test data
val_data = np.loadtxt(args.val_path, delimiter=',')
val_data = torch.tensor(val_data).type(torch.float32)
val_recon = forward(val_data, model, x_avg, x_std, args)
val_err = torch.sum((val_recon - val_data)**2, dim=1,
keepdim=True) # squared error
ve = val_err * error_scaler
test_data = np.loadtxt(args.test_path, delimiter=',')
test_lbl = test_data[:, -1]
data_len = test_data.shape[0] # retreive labels and length
test_data = torch.tensor(test_data[:, :-1]).type(torch.float32)
test_recon = forward(test_data, model, x_avg, x_std, args)
test_err = torch.sum((test_recon - test_data)**2, dim=1,
keepdim=True) # squared error
te = test_err * error_scaler
# 2. measure validation error and test error
neptune.set_property('validation error', torch.sum(ve).item())
neptune.set_property('test error', torch.sum(te).item())
# 2. plot reconstruction results
cols = ['sensor1', 'sensor2'] # features
ids_col = range(test_data.shape[0]) # for index
for j, (data, recon, str) in enumerate([(val_data, val_recon,
'Validation'),
(test_data, test_recon, 'Test')]):
fig, axs = plt.subplots(len(cols), 1, figsize=(12, 3))
for i, col in enumerate(cols):
axs[i].plot(ids_col,
data.numpy()[:, i],
'-c',
linewidth=2,
label='Raw Data')
axs[i].plot(ids_col,
recon.detach().numpy()[:, i],
'-b',
linewidth=1,
label='Reconstructed Data')
axs[1].legend() # only add legend for second row
fig.suptitle('Time Series of ' + str)
log_chart('Data-Reconstruction', fig)
#3. find threshold
T = (torch.mean(ve) + 2 * torch.std(ve)).item()
T_ = np.empty((data_len, 1))
T_[:] = T # for plotting threshold
if args.use_smoothing == 1:
ve = smooth(ve.detach().numpy(), args.window_size)
te = smooth(
te.detach().numpy(), args.window_size
) # smoothing removes first window_size samples. (Tx, 1) -> (Tx-args.window_size+1, 1)
pred = (te > T) # pred is classification result
pad = np.empty((args.window_size - 1, 1))
pad[:] = 0
pred = np.vstack([pad, pred]) # add 0 padding
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
acc = accuracy_score(test_lbl, pred)
neptune.set_property('acc', acc)
prec = precision_score(test_lbl, pred)
neptune.set_property('prec', prec)
rec = recall_score(test_lbl, pred)
neptune.set_property('rec', rec)
f1 = f1_score(test_lbl, pred)
neptune.set_property('f1', f1)
# 4. draw plot
fig = plt.figure(figsize=(24, 4))
ids = list(range(data_len))
pad[:] = np.nan
te = np.vstack([pad, te]) # add nan padding
plt.plot(ids[:ar[0]],
te[:ar[0]],
'-c',
label='Test Reconstruction Error (Normal)')
plt.plot(ids[ar[0]:ar[1]],
te[ar[0]:ar[1]],
'-r',
label='Test Reconstruction Error (Anomaly)')
plt.plot(ids[ar[1]:], te[ar[1]:], '-c')
plt.plot(ids, T_, '--b', label='Threshold')
plt.xlabel('Time')
plt.ylabel('Error')
plt.legend()
#plt.ylim((0,2E5))
plt.title('Reconstruction Error')
log_chart('Reconstruction Error', fig)
def train(train_dataloader, eval_dataloader, model, config, neptune_project, freeze_model=False):
"""
:param train_dataset: iterator on the training set
:param eval_dataset: iterator on the test set
:param model: instance of the model to train
:param processor: processor object used for evaluation
:param config: Config
:param freeze_model: whether or not to freeze BERT
"""
if config["resume_training"]:
# retrieving and updating already existing experiment
exp = neptune_project.get_experiments(id=config["neptune_id"])[0]
else:
# creating a neptune experiment
exp = neptune_project.create_experiment(name="{}_{}".format(config["model_type"], str(datetime.now())),
params=config,
upload_source_files=['*.py', "models/", "utils/"],
tags=[config["model_type"]] + config["tags"])
num_training_steps = len(train_dataloader) * config['num_train_epochs'] // config['gradient_accumulation_steps']
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
'weight_decay': config['weight_decay']},
{'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
warmup_steps = math.ceil(num_training_steps * config['warmup_ratio'])
config['warmup_steps'] = warmup_steps if config['warmup_steps'] else 0
optimizer = AdamW(optimizer_grouped_parameters, lr=config['learning_rate'], eps=config['adam_epsilon'])
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=config['warmup_steps'],
num_training_steps=num_training_steps)
# will freeze all the model parameters except the classification part
if freeze_model:
model.freeze_bert_encoder()
# optimization
if config['fp16']:
try:
from apex import amp
except ImportError:
raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
model, optimizer = amp.initialize(model, optimizer, opt_level=config['fp16_opt_level'])
# if running on multiple GPUs
if config['n_gpu'] > 1:
model = torch.nn.DataParallel(model)
logging.info("***** Running training *****")
logging.info(" Num examples = %d", len(train_dataloader))
logging.info(" Num Epochs = %d", config['num_train_epochs'])
logging.info(" Total train batch size = %d", config['train_batch_size'])
logging.info(" Gradient Accumulation steps = %d", config['gradient_accumulation_steps'])
logging.info(" Total optimization steps = %d", num_training_steps)
tr_loss, train_acc, logging_loss = 0.0, 0.0, 0.0
model.zero_grad()
start_epoch = int(config.get("previous_checkpoint", 0) / len(train_dataloader))
global_step = start_epoch * len(train_dataloader)
train_iterator = trange(start_epoch, int(config['num_train_epochs']), desc="Epoch")
# starting training
for epoch in train_iterator:
epoch_losses = []
for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
# avoiding to feed already seen batches when resuming training
if config["resume_training"] and global_step < config.get("previous_checkpoint") + 1:
global_step += 1
continue
model.train()
if 'distilbert' not in config['model_type']:
inputs = {'input_ids': batch["input_ids"].to(device),
'attention_mask': batch["input_mask"].to(device),
'token_type_ids': batch["token_type_ids"].to(device) if config['model_type'] in
['bert', 'xlnet'] else None,
'labels': batch["labels"]}
else:
inputs = {'input_ids': batch["input_ids"].to(device),
'attention_mask': batch["input_mask"].to(device),
'labels': batch["labels"].to(device)}
outputs = model(**inputs)
loss, logits = outputs[:2] # model outputs are always tuple in pytorch-transformers (see doc)
# handle multi-gpus run
if config["n_gpu"] > 1:
loss = loss.mean()
print("\r%f" % loss, end='')
epoch_losses.append(loss.item())
if config['task_name'] == "multi-label":
with torch.no_grad():
logits = logits.sigmoid()
train_acc += accuracy_thresh(logits, inputs["labels"])
else:
train_acc += get_accuracy(logits.detach().cpu().numpy(), batch[3].detach().cpu().numpy())
# gradient accumulation
if config['gradient_accumulation_steps'] > 1:
loss = loss / config['gradient_accumulation_steps']
# optimization
if config['fp16']:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), config['max_grad_norm'])
else:
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), config['max_grad_norm'])
tr_loss += loss.item()
if (step + 1) % config['gradient_accumulation_steps'] == 0:
optimizer.step()
scheduler.step() # Update learning rate schedule
model.zero_grad()
global_step += 1
if config['logging_steps'] > 0 and global_step % config['logging_steps'] == 0:
exp.log_metric(log_name='lr', y=scheduler.get_lr()[0], x=global_step)
exp.log_metric(log_name='train_loss', y=(tr_loss - logging_loss) / config['logging_steps'],
x=global_step)
exp.log_metric(log_name='train_acc', y=train_acc / config['logging_steps'], x=global_step)
logging_loss = tr_loss
train_acc = 0.0
if config['save_steps'] > 0 and global_step % config['save_steps'] == 0:
# Save model checkpoint
output_dir = os.path.join(config['output_dir'], 'checkpoint-{}'.format(global_step))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
model_to_save = model.module if hasattr(model,
'module') else model # Take care of distributed/parallel training
model_to_save.save_pretrained(output_dir)
logging.info("Saving model checkpoint to %s", output_dir)
exp.log_artifact(os.path.join(output_dir, "pytorch_model.bin"),
"pytorch_model_{}.bin".format(global_step))
# Log metrics
if config['evaluate_during_training']:
results = evaluate(eval_dataloader, model, config, epoch)
for key, value in results["scalars"].items():
exp.log_metric(log_name='eval_{}'.format(key), y=value, x=epoch)
if "labels_probs" in results["arrays"].keys():
labels_probs = results["arrays"]["labels_probs"]
for i in range(labels_probs.shape[0]):
fig = plt.figure(figsize=(15, 15))
sns.distplot(labels_probs[i], kde=False, bins=100)
plt.title("Probability boxplot for label {}".format(i))
log_chart(name="dist_label_{}_epoch_{}".format(i, epoch), chart=fig, experiment=exp)
plt.close("all")
return global_step, tr_loss / global_step
def log_study_info(study, experiment=None,
log_study=True,
log_charts=True,
log_optimization_history=False,
log_contour=False,
log_parallel_coordinate=False,
log_slice=False,
params=None):
"""Logs runs results and parameters to neptune.
Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric),
best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts
('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder.
Args:
study('optuna.study.Study'): Optuna study object after training is completed.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
log_study('bool'): Whether optuna study object should be logged as pickle. Default is True.
log_charts('bool'): Deprecated argument. Whether all optuna visualizations charts should be logged.
By default all charts are sent.
To not log any charts set log_charts=False.
If you want to log a particular chart change the argument for that chart explicitly.
For example log_charts=False and log_slice=True will log only the slice plot to Neptune.
log_optimization_history('bool'): Whether optuna optimization history chart should be logged. Default is True.
log_contour('bool'): Whether optuna contour plot should be logged. Default is True.
log_parallel_coordinate('bool'): Whether optuna parallel coordinate plot should be logged. Default is True.
log_slice('bool'): Whether optuna slice chart should be logged. Default is True.
params(`list`): List of parameters to be visualized. Default is all parameters.
Examples:
Initialize neptune_monitor::
import neptune
import neptunecontrib.monitoring.optuna as opt_utils
neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME')
neptune.create_experiment(name='optuna sweep')
neptune_callback = opt_utils.NeptuneCallback()
Run Optuna training passing monitor as callback::
...
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=100, callbacks=[neptune_callback])
opt_utils.log_study_info(study)
You can explore an example experiment in Neptune:
https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts
"""
import optuna.visualization as vis
_exp = experiment if experiment else neptune
_exp.log_metric('best_score', study.best_value)
_exp.set_property('best_parameters', study.best_params)
if log_charts:
message = """log_charts argument is depraceted and will be removed in future releases.
Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly.
"""
warnings.warn(message)
log_optimization_history = True
log_contour = True
log_parallel_coordinate = True
log_slice = True
if log_study:
pickle_and_log_artifact(study, 'study.pkl', experiment=_exp)
if log_optimization_history:
log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=_exp)
if log_contour:
log_chart(name='contour', chart=vis.plot_contour(study, params=params), experiment=_exp)
if log_parallel_coordinate:
log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=params), experiment=_exp)
if log_slice:
log_chart(name='slice', chart=vis.plot_slice(study, params=params), experiment=_exp)
# Log model weights
model.save('model')
neptune.log_artifact('model')
# Evaluate model
eval_metrics = model.evaluate(x_test, y_test, verbose=0)
for j, metric in enumerate(eval_metrics):
neptune.log_metric('test_{}'.format(model.metrics_names[j]), metric)
# Log predictions as table
y_pred_proba = model.predict(x_test)
y_pred = np.argmax(y_pred_proba, axis=1)
y_pred = y_pred
df = pd.DataFrame(
data={
'y_test': y_test,
'y_pred': y_pred,
'y_pred_probability': y_pred_proba.max(axis=1)
})
log_table('predictions', df)
# Log model performance visualizations
fig, ax = plt.subplots()
plot_roc(y_test, y_pred_proba, ax=ax)
log_chart('ROC', fig)
fig, ax = plt.subplots()
plot_precision_recall(y_test, y_pred_proba, ax=ax)
log_chart('precision recall', fig)
plt.close('all')
