def test_get_projects_with_given_namespace(self, _):
# given
api_projects = [a_project(), a_project()]
# and
backend = MagicMock()
leaderboard = MagicMock()
backend.get_projects.return_value = api_projects
backend.create_leaderboard_backend.return_value = leaderboard
# and
session = Session(backend=backend)
# and
custom_namespace = "custom_namespace"
# when
projects = session.get_projects(custom_namespace)
# then
expected_projects = OrderedDict((
custom_namespace + "/" + p.name,
Project(leaderboard, p.id, custom_namespace, p.name),
) for p in api_projects)
self.assertEqual(expected_projects, projects)
# and
backend.get_projects.assert_called_with(custom_namespace)
def test_get_projects_with_given_namespace(self, _):
# given
session = Session(API_TOKEN)
# and
api_projects = [a_project(), a_project()]
# and
session._client.get_projects.return_value = api_projects
# and
custom_namespace = 'custom_namespace'
# when
projects = session.get_projects(custom_namespace)
# then
expected_projects = {
custom_namespace + '/' + p.name: Project(session._client, p.id,
custom_namespace, p.name)
for p in api_projects
}
self.assertEqual(expected_projects, projects)
# and
session._client.get_projects.assert_called_with(custom_namespace)
def test_get_projects_with_given_namespace(self, _):
# given
api_projects = [a_project(), a_project()]
# and
backend = MagicMock()
backend.get_projects.return_value = api_projects
# and
session = Session(backend=backend)
# and
custom_namespace = 'custom_namespace'
# when
projects = session.get_projects(custom_namespace)
# then
expected_projects = OrderedDict(
(custom_namespace + '/' + p.name,
Project(backend, p.id, custom_namespace, p.name))
for p in api_projects)
self.assertEqual(expected_projects, projects)
# and
backend.get_projects.assert_called_with(custom_namespace)
def __init__(self, telegram_api_token, neptune_api_token):
self.session = Session(api_token=neptune_api_token)
self.updater = Updater(token=telegram_api_token)
self.dispatcher = self.updater.dispatcher
self.neptune_project = None
self.project_name = None
self.dispatcher.add_handler(CommandHandler('project', self.project, pass_args=True))
self.dispatcher.add_handler(CommandHandler('experiments', self.experiments, pass_args=True))
self.dispatcher.add_handler(CommandHandler('experiment', self.experiment, pass_args=True))
self.dispatcher.add_handler(MessageHandler(Filters.command, self.unknown))
def get_experiment_by_id(id: str) -> Experiment:
session = Session.with_default_backend(NEPTUNE_TOKEN)
project = session.get_project('richt3211/thesis')
exp: Experiment = project.get_experiments(id)[0]
return exp
def init(api_token=None, project_qualified_name=None):
if project_qualified_name is None:
project_qualified_name = os.getenv(envs.PROJECT_ENV_NAME)
# pylint: disable=global-statement
with __lock:
global session, project
session = Session(api_token=api_token)
if project_qualified_name is None:
raise MissingProjectQualifiedName()
project = session.get_project(project_qualified_name)
return project
def __init__(
self,
project: str,
experiment_ids: list,
tag: list = None,
state: str = None,
trajectory_length=50,
num_repeats=20,
):
self.session = Session(backend=HostedNeptuneBackend())
self.proj = self.session.get_project(project)
self.runners = {}
self.experiment_ids = experiment_ids
self.tag = tag
self.state = state
self.trajectory_length = trajectory_length
self.num_repeats = num_repeats
self._restore_runners()
self._create_param_df()
def init(project_qualified_name=None,
api_token=None,
proxies=None,
backend=None,
**kwargs):
"""Initialize `Neptune client library <https://github.com/neptune-ai/neptune-client>`_ to work with
specific project.
Authorize user, sets value of global variable ``project`` to :class:`~neptune.projects.Project` object
that can be used to create or list experiments, notebooks, etc.
Args:
project_qualified_name (:obj:`str`, optional, default is ``None``):
Qualified name of a project in a form of ``namespace/project_name``.
If ``None``, the value of ``NEPTUNE_PROJECT`` environment variable will be taken.
api_token (:obj:`str`, optional, default is ``None``):
User's API token. If ``None``, the value of ``NEPTUNE_API_TOKEN`` environment variable will be taken.
.. note::
It is strongly recommended to use ``NEPTUNE_API_TOKEN`` environment variable rather than
placing your API token in plain text in your source code.
proxies (:obj:`dict`, optional, default is ``None``):
Argument passed to HTTP calls made via the `Requests <https://2.python-requests.org/en/master/>`_ library.
For more information see their proxies
`section <https://2.python-requests.org/en/master/user/advanced/#proxies>`_.
.. note::
Only `http` and `https` keys are supported by all features.
.. deprecated :: 0.4.4
Instead, use:
.. code :: python3
from neptune import HostedNeptuneBackendApiClient
neptune.init(backend=HostedNeptuneBackendApiClient(proxies=...))
backend (:class:`~neptune.ApiClient`, optional, default is ``None``):
By default, Neptune client library sends logs, metrics, images, etc to Neptune servers:
either publicly available SaaS, or an on-premises installation.
You can also pass the default backend instance explicitly to specify its parameters:
.. code :: python3
from neptune import HostedNeptuneBackendApiClient
neptune.init(backend=HostedNeptuneBackendApiClient(...))
Passing an instance of :class:`~neptune.OfflineApiClient` makes your code run without communicating
with Neptune servers.
.. code :: python3
from neptune import OfflineApiClient
neptune.init(backend=OfflineApiClient())
.. note::
Instead of passing a ``neptune.OfflineApiClient`` instance as ``backend``, you can set an
environment variable ``NEPTUNE_BACKEND=offline`` to override the default behaviour.
Returns:
:class:`~neptune.projects.Project` object that is used to create or list experiments, notebooks, etc.
Raises:
`NeptuneMissingApiTokenException`: When ``api_token`` is None
and ``NEPTUNE_API_TOKEN`` environment variable was not set.
`NeptuneMissingProjectQualifiedNameException`: When ``project_qualified_name`` is None
and ``NEPTUNE_PROJECT`` environment variable was not set.
`InvalidApiKey`: When given ``api_token`` is malformed.
`Unauthorized`: When given ``api_token`` is invalid.
Examples:
.. code:: python3
# minimal invoke
neptune.init()
# specifying project name
neptune.init('jack/sandbox')
# running offline
neptune.init(backend=neptune.OfflineApiClient())
"""
_check_for_extra_kwargs(init.__name__, kwargs)
project_qualified_name = assure_project_qualified_name(
project_qualified_name)
# pylint: disable=global-statement
with __lock:
global session, project
if backend is None:
backend_name = os.getenv(envs.BACKEND)
backend = backend_factory(
backend_name=backend_name,
api_token=api_token,
proxies=proxies,
)
session = Session(backend=backend)
project = session.get_project(project_qualified_name)
return project
def test_should_accept_given_api_token(self, _):
# when
session = Session(API_TOKEN)
# then
self.assertEqual(API_TOKEN, session.credentials.api_token)
def test_should_take_default_credentials_from_env(self, _):
# when
session = Session()
# then
self.assertEqual(API_TOKEN, session.credentials.api_token)
def train(cfg):
"""
Train a video model for many epochs on train set and evaluate it on val set.
Args:
cfg (CfgNode): configs. Details can be found in
slowfast/config/defaults.py
"""
# Set random seed from configs.
np.random.seed(cfg.RNG_SEED)
torch.manual_seed(cfg.RNG_SEED)
# Setup logging format.
logging.setup_logging(cfg)
# Print config.
logger.info("Train with config:")
logger.info(pprint.pformat(cfg))
if du.get_rank()==0 and du.is_master_proc(num_gpus=cfg.NUM_GPUS):
writer = SummaryWriter(log_dir=cfg.OUTPUT_DIR)
else:
writer = None
if du.get_rank()==0 and du.is_master_proc(num_gpus=cfg.NUM_GPUS) and not cfg.DEBUG:
tags = []
if 'TAGS' in cfg and cfg.TAGS !=[]:
tags=list(cfg.TAGS)
neptune.set_project('Serre-Lab/motion')
######################
overrides = sys.argv[1:]
overrides_dict = {}
for i in range(len(overrides)//2):
overrides_dict[overrides[2*i]] = overrides[2*i+1]
overrides_dict['dir'] = cfg.OUTPUT_DIR
######################
if 'NEP_ID' in cfg and cfg.NEP_ID != "":
session = Session()
project = session.get_project(project_qualified_name='Serre-Lab/motion')
nep_experiment = project.get_experiments(id=cfg.NEP_ID)[0]
else:
nep_experiment = neptune.create_experiment (name=cfg.NAME,
params=overrides_dict,
tags=tags)
else:
nep_experiment=None
# Build the video model and print model statistics.
model = build_model(cfg)
if du.is_master_proc(num_gpus=cfg.NUM_GPUS):
misc.log_model_info(model, cfg, is_train=True)
# Construct the optimizer.
optimizer = optim.construct_optimizer(model, cfg)
# Load a checkpoint to resume training if applicable.
if cfg.TRAIN.AUTO_RESUME and cu.has_checkpoint(cfg.OUTPUT_DIR):
logger.info("Load from last checkpoint.")
last_checkpoint = cu.get_last_checkpoint(cfg.OUTPUT_DIR)
checkpoint_epoch = cu.load_checkpoint(
last_checkpoint, model, cfg.NUM_GPUS > 1, optimizer
)
start_epoch = checkpoint_epoch + 1
elif cfg.TRAIN.CHECKPOINT_FILE_PATH != "":
logger.info("Load from given checkpoint file.")
checkpoint_epoch = cu.load_checkpoint(
cfg.TRAIN.CHECKPOINT_FILE_PATH,
model,
cfg.NUM_GPUS > 1,
optimizer,
inflation=cfg.TRAIN.CHECKPOINT_INFLATE,
convert_from_caffe2=cfg.TRAIN.CHECKPOINT_TYPE == "caffe2",
)
start_epoch = checkpoint_epoch + 1
else:
start_epoch = 0
# Create the video train and val loaders.
train_loader = loader.construct_loader(cfg, "train")
val_loader = loader.construct_loader(cfg, "val")
# Create meters.
if cfg.DETECTION.ENABLE:
train_meter = AVAMeter(len(train_loader), cfg, mode="train")
val_meter = AVAMeter(len(val_loader), cfg, mode="val")
else:
train_meter = TrainMeter(len(train_loader), cfg)
val_meter = ValMeter(len(val_loader), cfg)
# Perform the training loop.
logger.info("Start epoch: {}".format(start_epoch + 1))
for cur_epoch in range(start_epoch, cfg.SOLVER.MAX_EPOCH):
# Shuffle the dataset.
loader.shuffle_dataset(train_loader, cur_epoch)
# Train for one epoch.
train_epoch(train_loader, model, optimizer, train_meter, cur_epoch, writer, nep_experiment, cfg)
# Compute precise BN stats.
# if cfg.BN.USE_PRECISE_STATS and len(get_bn_modules(model)) > 0:
# calculate_and_update_precise_bn(
# train_loader, model, cfg.BN.NUM_BATCHES_PRECISE
# )
# Save a checkpoint.
if cu.is_checkpoint_epoch(cur_epoch, cfg.TRAIN.CHECKPOINT_PERIOD):
cu.save_checkpoint(cfg.OUTPUT_DIR, model, optimizer, cur_epoch, cfg)
# Evaluate the model on validation set.
if misc.is_eval_epoch(cfg, cur_epoch):
eval_epoch(val_loader, model, val_meter, cur_epoch, nep_experiment, cfg)
if du.get_rank()==0 and du.is_master_proc(num_gpus=cfg.NUM_GPUS) and not cfg.DEBUG:
nep_experiment.log_metric('epoch', cur_epoch)
def run(self, **kwargs):
""" Run the closed loop experiment cycle
Parameters
----------
save_freq : int, optional
The frequency with which to checkpoint the state of the optimization. Defaults to None.
save_at_end : bool, optional
Save the state of the optimization at the end of a run, even if it is stopped early.
Default is True.
save_dir : str, optional
The directory to save checkpoints locally. Defaults to `~/.summit/runner`.
"""
# Set parameters
prev_res = None
self.restarts = 0
n_objs = len(self.experiment.domain.output_variables)
fbest_old = np.zeros(n_objs)
fbest = np.zeros(n_objs)
# Serialization
save_freq = kwargs.get("save_freq")
save_dir = kwargs.get("save_dir", str(get_summit_config_path()))
self.uuid_val = uuid.uuid4()
save_dir = pathlib.Path(save_dir) / "runner" / str(self.uuid_val)
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
save_at_end = kwargs.get("save_at_end", True)
# Create neptune experiment
if self.neptune_exp is None:
session = Session(backend=HostedNeptuneBackend())
proj = session.get_project(self.neptune_project)
neptune_exp = proj.create_experiment(
name=self.neptune_experiment_name,
description=self.neptune_description,
upload_source_files=self.neptune_files,
logger=self.logger,
tags=self.neptune_tags,
)
else:
neptune_exp = self.neptune_exp
# Run optimization loop
for i in progress_bar(range(self.max_iterations)):
# Get experiment suggestions
if i == 0:
k = self.n_init if self.n_init is not None else self.batch_size
next_experiments = self.strategy.suggest_experiments(
num_experiments=k)
else:
next_experiments = self.strategy.suggest_experiments(
num_experiments=self.batch_size, prev_res=prev_res)
prev_res = self.experiment.run_experiments(next_experiments)
# Send best objective values to Neptune
for j, v in enumerate(self.experiment.domain.output_variables):
if i > 0:
fbest_old[j] = fbest[j]
if v.maximize:
fbest[j] = self.experiment.data[v.name].max()
elif not v.maximize:
fbest[j] = self.experiment.data[v.name].min()
neptune_exp.send_metric(v.name + "_best", fbest[j])
# Send hypervolume for multiobjective experiments
if n_objs > 1:
output_names = [
v.name for v in self.experiment.domain.output_variables
]
data = self.experiment.data[output_names].copy()
for v in self.experiment.domain.output_variables:
if v.maximize:
data[(v.name, "DATA")] = -1.0 * data[v.name]
y_pareto, _ = pareto_efficient(data.to_numpy(), maximize=False)
hv = hypervolume(y_pareto, self.ref)
neptune_exp.send_metric("hypervolume", hv)
# Save state
if save_freq is not None:
file = save_dir / f"iteration_{i}.json"
if i % save_freq == 0:
self.save(file)
neptune_exp.send_artifact(str(file))
if not save_dir:
os.remove(file)
# Stop if no improvement
compare = np.abs(fbest - fbest_old) > self.f_tol
if all(compare) or i <= 1:
nstop = 0
else:
nstop += 1
if self.max_same is not None:
if nstop >= self.max_same and self.restarts >= self.max_restarts:
self.logger.info(
f"{self.strategy.__class__.__name__} stopped after {i+1} iterations and {self.restarts} restarts."
)
break
elif nstop >= self.max_same:
nstop = 0
prev_res = None
self.strategy.reset()
self.restarts += 1
# Save at end
if save_at_end:
file = save_dir / f"iteration_{i}.json"
self.save(file)
neptune_exp.send_artifact(str(file))
if not save_dir:
os.remove(file)
# Stop the neptune experiment
neptune_exp.stop()
class PlotExperiments:
""" Make plots from benchmarks tracked on Neptune
Parameters
----------
project : str, optional
The name of the Neptune.ai project
experiment_ids : list of str, optional
A list of experiment ids to pull from Neptune.ai
csv_filename : str, optional
Name of the CSV filename
tag : list of str, optional
A list of tags used as filters
state : str, optional
The state of the experiments. Must be succeeded, failed, running or aborted.
trajectory_length : int, optional
The maximum number of iterations for each experiment. Defaults to 50.
num_repeats : int, optional
The number of repeats required for each hyperparameter combination.s
"""
def __init__(
self,
project: str = None,
experiment_ids: list = None,
csv_filename: str = None,
tag: list = None,
state: list = None,
trajectory_length=50,
num_repeats=20,
):
self.session = Session(backend=HostedNeptuneBackend())
self.proj = self.session.get_project(project)
self.runners = {}
self.experiment_ids = experiment_ids
self.tag = tag
self.state = state
self.trajectory_length = trajectory_length
self.num_repeats = num_repeats
self._restore_runners()
self._create_param_df()
def _restore_runners(self):
"""Restore runners from Neptune Artifacts"""
# Download artifacts
n_experiments = len(self.experiment_ids)
experiments = []
if n_experiments > 100:
for i in range(n_experiments // 100):
experiments += self.proj.get_experiments(
id=self.experiment_ids[i * 100:(i + 1) * 100],
tag=self.tag,
state=self.state,
)
remainder = n_experiments % 100
experiments += self.proj.get_experiments(
id=self.experiment_ids[(i + 1) * 100:(i + 1) * 100 +
remainder],
tag=self.tag,
state=self.state,
)
else:
experiments = self.proj.get_experiments(id=self.experiment_ids,
tag=self.tag,
state=self.state)
for experiment in experiments:
path = f"data/{experiment.id}"
try:
os.mkdir(path, )
except FileExistsError:
pass
experiment.download_artifacts(destination_dir=path)
# Unzip somehow
files = os.listdir(path)
with zipfile.ZipFile(path + "/" + files[0], "r") as zip_ref:
zip_ref.extractall(path)
# Get filename
path += "/output"
files = os.listdir(path)
files_json = [f for f in files if ".json" in f]
if len(files_json) == 0:
warnings.warn(f"{experiment.id} has no file attached.")
continue
# Restore runner
r = Runner.load(path + "/" + files_json[0])
self.runners[experiment.id] = r
# Remove file
shutil.rmtree(f"data/{experiment.id}")
def _create_param_df(self, reference=[-2957, 10.7]):
"""Create a parameters dictionary
Parameters
----------
reference : array-like, optional
Reference for the hypervolume calculatio
"""
records = []
for experiment_id, r in self.runners.items():
record = {}
record["experiment_id"] = experiment_id
# Transform
transform_name = r.strategy.transform.__class__.__name__
transform_params = r.strategy.transform.to_dict(
)["transform_params"]
record["transform_name"] = transform_name
if transform_name == "Chimera":
hierarchy = transform_params["hierarchy"]
for objective_name, v in hierarchy.items():
key = f"{objective_name}_tolerance"
record[key] = v["tolerance"]
elif transform_name == "MultitoSingleObjective":
record.update(transform_params)
# Strategy
record["strategy_name"] = r.strategy.__class__.__name__
# Batch size
record["batch_size"] = r.batch_size
# Number of initial experiments
try:
record["num_initial_experiments"] = r.n_init
except AttributeError:
pass
# Terminal hypervolume
data = r.experiment.data[["sty", "e_factor"]].to_numpy()
data[:, 0] *= -1 # make it a minimzation problem
y_front, _ = pareto_efficient(data[:self.trajectory_length, :],
maximize=False)
hv = hypervolume(y_front, ref=reference)
record["terminal_hypervolume"] = hv
# Computation time
time = (r.experiment.data["computation_t"].
iloc[0:self.trajectory_length].sum())
record["computation_t"] = time
record["noise_level"] = r.experiment.noise_level
records.append(record)
# Make pandas dataframe
self.df = pd.DataFrame.from_records(records)
return self.df
def _create_label(self, unique):
transform_text = unique["transform_name"]
chimera_params = f" (STY tol.={unique['sty_tolerance']}, E-factor tol.={unique['e_factor_tolerance']})"
transform_text += (chimera_params
if unique["transform_name"] == "Chimera" else "")
return f"{unique['strategy_name']}, {transform_text}, {unique['num_initial_experiments']} initial experiments"
def best_pareto_grid(self, ncols=3, figsize=(20, 40)):
""" Make a grid of pareto plots
Only includes the run with the maximum terminal hypervolume for each
unique hyperparameter combination.
Parameters
----------
ncols : int, optional
The number of columns in the grid. Defaults to 3
figsize : tuple, optional
The figure size. Defaults to 20 wide x 40 high
"""
# Group experiment repeats
df = self.df.copy()
df = df.set_index("experiment_id")
df = df.drop(columns=["terminal_hypervolume", "computation_t"])
uniques = df.drop_duplicates(keep="last") # This actually groups them
uniques = uniques.sort_values(by=["strategy_name", "transform_name"])
df_new = self.df.copy()
nrows = len(uniques) // ncols
nrows += 1 if len(uniques) % ncols != 0 else 0
fig = plt.figure(figsize=figsize)
fig.subplots_adjust(wspace=0.2, hspace=0.5)
i = 1
# Loop through groups of repeats
for index, unique in uniques.iterrows():
# Find number of matching rows to this unique row
temp_df = df_new.merge(unique.to_frame().transpose(), how="inner")
# Find experiment with maximum hypervolume
max_hv_index = temp_df["terminal_hypervolume"].argmax()
experiment_id = temp_df.iloc[max_hv_index]["experiment_id"]
# Get runner
r = self.runners[experiment_id]
# Create pareto plot
ax = plt.subplot(nrows, ncols, i)
old_data = r.experiment._data.copy()
r.experiment._data = r.experiment.data.iloc[:self.
trajectory_length, :]
r.experiment.pareto_plot(ax=ax)
r.experiment._data = old_data
title = self._create_label(unique)
title = "\n".join(wrap(title, 30))
ax.set_title(title)
ax.set_xlabel(r"Space Time Yield ($kg \; m^{-3} h^{-1}$)")
ax.set_ylabel("E-factor")
ax.set_xlim(0.0, float(1.2e4))
ax.set_ylim(0.0, 300.0)
ax.ticklabel_format(axis="x", style="scientific")
i += 1
return fig
def plot_hv_trajectories(
self,
reference=[-2957, 10.7],
plot_type="matplotlib",
include_experiment_ids=False,
min_terminal_hv_avg=0,
ax=None,
):
""" Plot the hypervolume trajectories with repeats as 95% confidence interval
Parameters
----------
reference : array-like, optional
Reference for the hypervolume calculation. Defaults to -2957, 10.7
plot_type : str, optional
Plotting backend to use: matplotlib or plotly. Defaults to matplotlib.
include_experiment_ids : bool, optional
Whether to include experiment ids in the plot labels
min_terminal_hv_avg : float, optional`
Minimum terminal average hypervolume cutoff for inclusion in the plot. Defaults to 0.
"""
# Create figure
if plot_type == "matplotlib":
if ax is not None:
fig = None
else:
fig, ax = plt.subplots(1)
elif plot_type == "plotly":
fig = go.Figure()
else:
raise ValueError(
f"{plot_type} is not a valid plot type. Must be matplotlib or plotly."
)
# Group experiment repeats
df = self.df.copy()
df = df.set_index("experiment_id")
df = df.drop(columns=["terminal_hypervolume", "computation_t"])
uniques = df.drop_duplicates(keep="last") # This actually groups them
df_new = self.df.copy()
if plot_type == "plotly":
colors = px.colors.qualitative.Plotly
else:
colors = COLORS
cycle = len(colors)
c_num = 0
self.hv = {}
for index, unique in uniques.iterrows():
# Find number of matching rows to this unique row
temp_df = df_new.merge(unique.to_frame().transpose(), how="inner")
ids = temp_df["experiment_id"].values
# Calculate hypervolume trajectories
ids = ids if len(ids) < self.num_repeats else ids[:self.
num_repeats]
hv_trajectories = np.zeros([self.trajectory_length, len(ids)])
for j, experiment_id in enumerate(ids):
r = self.runners[experiment_id]
data = r.experiment.data[["sty", "e_factor"]].to_numpy()
data[:, 0] *= -1 # make it a minimzation problem
for i in range(self.trajectory_length):
y_front, _ = pareto_efficient(data[0:i + 1, :],
maximize=False)
hv_trajectories[i, j] = hypervolume(y_front, ref=reference)
# Mean and standard deviation
hv_mean_trajectory = np.mean(hv_trajectories, axis=1)
hv_std_trajectory = np.std(hv_trajectories, axis=1)
if hv_mean_trajectory[-1] < min_terminal_hv_avg:
continue
# Update plot
t = np.arange(1, self.trajectory_length + 1)
# label = self._create_label(unique)
transform = unique["transform_name"]
if transform == "MultitoSingleObjective":
transform = "Custom"
label = (f"""{unique["strategy_name"]} ({transform})"""
if transform is not "Transform" else
f"""{unique["strategy_name"]}""")
if include_experiment_ids:
label += f" ({ids[0]}-{ids[-1]})"
lower = hv_mean_trajectory - 1.96 * hv_std_trajectory
lower = np.clip(lower, 0, None)
upper = hv_mean_trajectory + 1.96 * hv_std_trajectory
if plot_type == "matplotlib":
ax.plot(t,
hv_mean_trajectory,
label=label,
color=colors[c_num])
ax.fill_between(t,
lower,
upper,
alpha=0.1,
color=colors[c_num])
elif plot_type == "plotly":
r, g, b = hex_to_rgb(colors[c_num])
color = lambda alpha: f"rgba({r},{g},{b},{alpha})"
fig.add_trace(
go.Scatter(
x=t,
y=hv_mean_trajectory,
mode="lines",
name=label,
line=dict(color=color(1)),
legendgroup=label,
))
fig.add_trace(
go.Scatter(
x=t,
y=lower,
mode="lines",
fill="tonexty",
line=dict(width=0),
fillcolor=color(0.1),
showlegend=False,
legendgroup=label,
))
fig.add_trace(
go.Scatter(
x=t,
y=upper,
mode="lines",
fill="tozeroy",
line=dict(width=0),
fillcolor=color(0.1),
showlegend=False,
legendgroup=label,
))
if cycle == c_num + 1:
c_num = 0
elif plot_type == "plotly":
c_num += 1
elif plot_type == "matplotlib":
c_num += 2
# Plot formattting
if plot_type == "matplotlib":
ax.set_xlabel("Experiments")
ax.set_ylabel("Hypervolume")
legend = ax.legend(loc="upper left")
ax.tick_params(direction="in")
ax.set_xlim(1, self.trajectory_length)
if fig is None:
return ax, legend
else:
return fig, ax, legend
elif plot_type == "plotly":
fig.update_layout(xaxis=dict(title="Experiments"),
yaxis=dict(title="Hypervolume"))
fig.show()
return fig
def _create_label(self, unique):
transform_text = (unique["transform_name"]
if unique["transform_name"] != "Transform" else
"No transform")
chimera_params = f" (STY tol.={unique['sty_tolerance']}, E-factor tol.={unique['e_factor_tolerance']})"
transform_text += (chimera_params
if unique["transform_name"] == "Chimera" else "")
final_text = f"{unique['strategy_name']}, {transform_text}, {unique['num_initial_experiments']} initial experiments"
if unique["num_initial_experiments"] == 1:
final_text = final_text.rstrip("s")
return final_text
def time_hv_bar_plot(self, ax=None):
"""Plot average CPU time and terminal hypervolume for hyperparameter combinations
Only shows the hyperparemter combination with the highest mean terminal hypervolume
for each strategy
Parameters
----------
ax : `matplotlib.pyplot.axes`, optional
Matplotlib axis for plotting
"""
df = self.df
# Group repeats
by = [
"strategy_name",
"transform_name",
"sty_tolerance",
"e_factor_tolerance",
"batch_size",
"num_initial_experiments",
]
grouped_df = (df.groupby(
by=by,
dropna=False,
).head(self.num_repeats).groupby(by=by, dropna=False))
# Mean and std deviation
means = grouped_df.mean()
stds = grouped_df.std()
# Find the maximum terminal hypervolume for each strategy
indices = means.groupby(
by=["strategy_name"]).idxmax()["terminal_hypervolume"]
means = means.loc[indices]
stds = stds.loc[indices]
# Ascending hypervolume
ordered_indices = means["terminal_hypervolume"].argsort()
means = means.iloc[ordered_indices]
stds = stds.iloc[ordered_indices]
# Convert to per iteration
means["computation_t_iter"] = means[
"computation_t"] / self.trajectory_length
stds["computation_t_iter"] = stds[
"computation_t"] / self.trajectory_length
# Clip std deviations to not be negative
stds["terminal_hypervolume"] = stds["terminal_hypervolume"].clip(
0, means["terminal_hypervolume"])
stds["computation_t_iter"] = stds["computation_t_iter"].clip(
0, means["computation_t_iter"])
# Rename
rename = {
"terminal_hypervolume": "Terminal hypervolume",
"computation_t_iter": "Computation time per iteration",
}
means = means.rename(columns=rename)
stds = stds.rename(columns=rename)
# Bar plot
ax = means.plot(
kind="bar",
colormap=CMAP,
y=["Terminal hypervolume", "Computation time per iteration"],
secondary_y="Computation time per iteration",
yerr=stds,
capsize=4,
mark_right=False,
ax=ax,
alpha=0.9,
)
strategy_names = means.index.to_frame()["strategy_name"].values
transform_names = means.index.to_frame()["transform_name"].values
for i, t in enumerate(transform_names):
if t == "Transform":
transform_names[i] = ""
elif t == "MultitoSingleObjective":
transform_names[i] = "Custom"
ax.set_xticklabels([
f"{s} \n ({t})" if t is not "" else f"{s}"
for s, t in zip(strategy_names, transform_names)
])
ax.set_xlabel("")
ax.right_ax.set_ylabel("Time (seconds)")
ax.right_ax.set_yscale("log")
ax.set_ylabel("Hypervolume")
plt.minorticks_off()
for tick in ax.get_xticklabels():
tick.set_rotation(45)
return ax
def parallel_plot(self, experiment_id, ax=None):
# Get data
r = self.runners[experiment_id]
data = r.experiment.data
labels = [
"Conc. 4",
"Equiv. 5",
"Temperature",
r"$\tau$",
"E-factor",
"STY",
]
columns = [
"conc_dfnb",
"equiv_pldn",
"temperature",
"tau",
"e_factor",
"sty",
]
data = data[columns]
# Standardize data
mins = data.min().to_numpy()
maxs = data.max().to_numpy()
data_std = (data - mins) / (maxs - mins)
data_std["experiments"] = np.arange(1, data_std.shape[0] + 1)
# Creat plot
if ax is not None:
fig = None
else:
fig, ax = plt.subplots(1, figsize=(10, 5))
# color map
new_colors = np.flip(COLORS, axis=0)
new_cmap = ListedColormap(new_colors[5:])
# Plot data
parallel_coordinates(
data_std,
"experiments",
cols=columns,
colormap=new_cmap,
axvlines=False,
alpha=0.4,
)
# Format plot (colorbar, labels, etc.)
bounds = np.linspace(1, data_std.shape[0] + 1, 6)
bounds = bounds.astype(int)
cax, _ = mpl.colorbar.make_axes(ax)
cb = mpl.colorbar.ColorbarBase(
cax,
cmap=new_cmap,
spacing="proportional",
ticks=bounds,
boundaries=bounds,
label="Number of Experiments",
)
title = r.strategy.__class__.__name__
ax.set_title(title)
ax.set_xticklabels(labels)
for side in ["left", "right", "top", "bottom"]:
ax.spines[side].set_visible(False)
ax.grid(alpha=0.5, axis="both")
ax.tick_params(length=0)
ax.get_legend().remove()
for tick in ax.get_xticklabels():
tick.set_rotation(45)
if fig is None:
return ax
else:
return fig, ax
def iterations_to_threshold(self,
sty_threshold=1e4,
e_factor_threshold=10.0):
# Group experiment repeats
df = self.df.copy()
df = df.set_index("experiment_id")
uniques = df.drop_duplicates(keep="last") # This actually groups them
df_new = self.df.copy()
experiments = {}
results = []
uniques["mean_iterations"] = None
uniques["std_iterations"] = None
uniques["num_repeats"] = None
# Find iterations to threshold
for index, unique in uniques.iterrows():
# Find number of matching rows to each unique row
temp_df = df_new.merge(unique.to_frame().transpose(), how="inner")
ids = temp_df["experiment_id"].values
# Number of iterations calculation
num_iterations = []
something_happens = False
for experiment_id in ids:
data = self.runners[experiment_id].experiment.data[[
"sty", "e_factor"
]]
# Check if repeat matches threshold requirements
meets_threshold = data[
(data["sty"] >= sty_threshold)
& (data["e_factor"] <= e_factor_threshold)]
# Calculate iterations to meet threshold
if len(meets_threshold.index) > 0:
num_iterations.append(meets_threshold.index[0])
something_happens = True
if something_happens:
mean = np.mean(num_iterations)
std = np.std(num_iterations)
uniques["mean_iterations"][index] = mean
uniques["std_iterations"][index] = std
uniques["num_repeats"][index] = len(num_iterations)
return uniques
class TelegramBot:
def __init__(self, telegram_api_token, neptune_api_token):
self.session = Session(api_token=neptune_api_token)
self.updater = Updater(token=telegram_api_token)
self.dispatcher = self.updater.dispatcher
self.neptune_project = None
self.project_name = None
self.dispatcher.add_handler(CommandHandler('project', self.project, pass_args=True))
self.dispatcher.add_handler(CommandHandler('experiments', self.experiments, pass_args=True))
self.dispatcher.add_handler(CommandHandler('experiment', self.experiment, pass_args=True))
self.dispatcher.add_handler(MessageHandler(Filters.command, self.unknown))
def run(self):
self.updater.start_polling()
def project(self, bot, update, args):
if args:
if args[0] == 'select':
self._project_select(bot, update, args)
elif args[0] == 'list':
self._project_list(bot, update, args)
else:
self._project_help(bot, update)
else:
self._project_help(bot, update)
def experiments(self, bot, update, args):
if not self.neptune_project:
self._no_project_selected(bot, update)
else:
if args:
if args[0] == 'last':
self._experiments_last(bot, update, args)
elif args[0] == 'best':
self._experiments_best(bot, update, args)
elif args[0] == 'state':
self._experiments_state(bot, update, args)
else:
self._experiments_help(bot, update)
else:
self._experiments_help(bot, update)
def experiment(self, bot, update, args):
if not self.neptune_project:
self._no_project_selected(bot, update)
else:
if args:
if args[0] == 'link':
self._experiment_link(bot, update, args)
elif args[0] == 'plot':
self._experiment_plot(bot, update, args)
else:
self._experiment_help(bot, update)
else:
self._experiment_help(bot, update)
def unknown(self, bot, update):
bot.send_message(chat_id=update.message.chat_id,
text="Sorry, I only undestand /project, /experiments, /experiment")
def _project_list(self, bot, update, args):
if len(args) != 2:
msg = ['message should have a format:',
'/project list NAMESPACE',
'for example:',
'/project list neptune-ai']
msg = '\n'.join(msg)
else:
namespace = args[1]
project_names = self.session.get_projects(namespace).keys()
msg = '\n'.join(project_names)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _project_select(self, bot, update, args):
if len(args) != 2:
msg = ['message should have a format:',
'/project select NAMESPACE/PROJECT_NAME',
'for example:',
'/project select neptune-ai/neptune-examples']
msg = '\n'.join(msg)
else:
self.project_name = args[1]
namespace = self.project_name.split('/')[0]
self.neptune_project = self.session.get_projects(namespace)[self.project_name]
msg = 'Selected a project: {}'.format(self.project_name)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _project_help(self, bot, update):
msg = """Available options are:\n
/project list NAMESPACE
/project select NAMESPACE/PROJECT_NAME
"""
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiments_last(self, bot, update, args):
if len(args) != 3:
msg = ['message should have a format:',
'/experiments last NR_EXPS TIMESTAMP',
'for example:',
'/experiments last 5 finished']
msg = '\n'.join(msg)
else:
nr_exps = int(args[1])
timestamp = args[2]
if timestamp not in ['created', 'finished']:
msg = 'choose created or finished as timestamp'
else:
leaderboard = self.neptune_project.get_leaderboard()
leaderboard[timestamp] = pd.to_datetime(leaderboard[timestamp])
leaderboard.sort_values(timestamp, ascending=False, inplace=True)
ids = leaderboard['id'].tolist()[:nr_exps]
ids = ['id'] + ids
msg = '\n'.join(ids)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiments_best(self, bot, update, args):
if len(args) != 3:
msg = ['message should have a format:',
'/experiments best METRIC NR_EXPS',
'for example:',
'/experiments best log_loss 3']
msg = '\n'.join(msg)
else:
metric_name = args[1]
metric_column = 'channel_' + metric_name
nr_exps = int(args[2])
leaderboard = self.neptune_project.get_leaderboard()
scores = leaderboard.sort_values([metric_column], ascending=False)[['id', metric_column]]
msg = 'id | {}\n'.format(metric_name)
for idx, metric in scores.values[:nr_exps]:
msg = msg + '{} | {}\n'.format(idx, metric)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiments_state(self, bot, update, args):
if len(args) != 3:
msg = ['message should have a format:',
'/experiments state STATE NR_EXPS',
'for example:',
'/experiments state running 4']
msg = '\n'.join(msg)
else:
state = args[1]
nr_exps = int(args[2])
leaderboard = self.neptune_project.get_leaderboard(state=state)
leaderboard['created'] = pd.to_datetime(leaderboard['created'])
leaderboard.sort_values('created', ascending=False, inplace=True)
ids = leaderboard['id'].tolist()[:nr_exps]
ids = ['id'] + ids
msg = '\n'.join(ids)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiments_help(self, bot, update):
msg = """Available options are:\n
/experiments last NR_EXPS
/experiments best METRIC_NAME NR_EXPS(optional)
/experiments state STATE NR_EXPS(optional)
"""
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiment_link(self, bot, update, args):
if len(args) != 2:
msg = ['message should have a format:',
'/experiment link SHORT_ID',
'for example:',
'/experiment link NEP-508']
msg = '\n'.join(msg)
else:
short_id = args[1]
namespace, project = self.project_name.split('/')
msg = 'https://ui.neptune.ai/o/{}/org/{}/e/{}/details'.format(namespace, project, short_id)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiment_plot(self, bot, update, args):
if len(args) < 3:
msg = ['message should have a format:',
'/experiment plot SHORT_ID METRIC_NAME OTHER_METRIC_NAME',
'for example:',
'/experiment plot NEP-508 train_loss valid_loss']
msg = '\n'.join(msg)
bot.send_message(chat_id=update.message.chat_id, text=msg)
else:
short_id = args[1]
if len(args) == 2:
metric_names = [args[2]]
else:
metric_names = args[2:]
experiment = self.neptune_project.get_experiments(id=short_id)[0]
data = experiment.get_numeric_channels_values(*metric_names)
fig = plt.figure()
for channel_name in data.columns:
if channel_name != 'x':
plt.plot('x', channel_name, data=data,
marker='', linewidth=2, label=channel_name)
plt.legend()
buffer = BytesIO()
fig.savefig(buffer, format='png')
buffer.seek(0)
update.message.reply_photo(buffer)
def _experiment_help(self, bot, update):
msg = """Available options are:\n
/experiment link SHORT_ID
/experiments plot SHORT_ID METRIC_NAME (OTHER_METRIC_NAME)
"""
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _no_project_selected(self, bot, update):
msg = ["You haven't selected your project.",
"Do so by running:\n"
"/project select NAMESPACE/PROJECT_NAME",
"For example:",
"/project select neptune-ai/neptune-examples"]
msg = '\n'.join(msg)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def run():
# Parse arguments.
#
(experiment_name,
param_path,
work_dir,
data_path,
copy_directive,
data_overrides,
data_root_dir,
x_files,
y_files,
x_val_files,
y_val_files,
repo_dir,
param_overrides,
log_level,
random_seed,
num_workers,
continue_experiment,
copy_patches,
multi_gpu,
create_stats,
generator_flag,
neptune_project) = collect_arguments()
###################################################################################
# Initialization: Initialize logging, data-loaders and model.
###################################################################################
# Extend experiment commander with utility functions.
#
ExperimentCommander.init_head_report = init_head_report
ExperimentCommander.log_normalized_head_report = log_normalized_head_report
# Initialize experiment commander object.
#
cmd = ExperimentCommander(experiment_name=experiment_name,
work_dir_path=work_dir,
data_file_path=None,
data_overrides=data_overrides,
data_copy_config=copy_directive,
param_file_path=param_path,
param_overrides=param_overrides,
create_stats=create_stats,
continue_experiment=continue_experiment,
repository_root_dir=repo_dir,
file_log_level=log_level,
seed=random_seed)
# Create config object from configuration file.
#
config = DigitalPathologyConfiguration(cmd.get_parameter_path(), param_overrides=cmd.get_param_overrides())
# Set up neptune experiment.
#
if cmd.existing_neptune_experiment() and continue_experiment:
project_name, experiment_id = cmd.get_experiment_project_id()
session = Session() # with_default_backend()
project = session.get_project(project_qualified_name=project_name)
experiment = project.get_experiments(id=experiment_id)[0]
cmd.logger.info('Succesfully loaded an existing neptune experiment.')
else:
cmd.logger.info('Initialising neptune experiment.')
neptune.init(project_qualified_name=neptune_project)
experiment = neptune.create_experiment(name=experiment_name,
params=config.get_complete_config_dict(),
logger=cmd.logger)
cmd.save_experiment_project_id(neptune_project, experiment.id)
# Initialize, configure and build the model.
#
model = get_model(config, multi_gpu)
cmd.logger.info('Done initialising {} model.'.format(config.model['type']))
# Print model summary.
#
if config.model['type'] == 'deepensemble' and config.model['ensemble mode'] != 'single':
summary(model._model_instance[0], input_size=[config.model['input shape']])
else:
summary(model._model_instance, input_size=[config.model['input shape']])
# Initialize state.
#
state = DigitalPathologyState(plateau_metric=config.training['metric name'],
plateau_metric_higher_is_better=config.training['higher is better'],
plateau_averaging_length=config.training['averaging length'])
# Try to load previous checkpoint, if exists.
#
try:
cmd.load(model, state)
cmd.logger.info('Succesfully loaded a previous checkpoint.')
except FileNotFoundError:
cmd.logger.info('No previous checkpoint available.')
pass
# Initialize training and validation dataloaders.
#
train_loader, valid_loader = get_generators(config=config,
state=state,
data_path=data_path,
param_path=param_path,
x_files=x_files,
y_files=y_files,
x_val_files=x_val_files,
y_val_files=y_val_files,
create_stats=create_stats,
copy_directive=copy_directive,
data_overrides=data_overrides,
num_workers=num_workers,
generator_flag=generator_flag,
copy_patches=copy_patches,
logger=cmd.logger)
# Initiate csv logger.
#
cmd.init_csv_logger(model.metricnames())
# Initialize head report.
#
cmd.init_head_report()
###################################################################################
# Training stage: Train and validate model.
###################################################################################
cmd.logger.info('Initialising training.')
for epoch in range(state['epoch'], config.model['epochs']):
# Update learning rate based on predetermined scheme.
#
if epoch in config.model['learning rate scheme']:
cmd.logger.info('Going to update learning rate to: {}'.format(config.model['learning rate scheme'][epoch]))
model.updatelearningrate(config.model['learning rate scheme'][epoch])
# Training epoch.
#
training_stats = Counter()
for i, (data, target) in enumerate(train_loader):
if isinstance(train_loader, torch.utils.data.DataLoader):
# Manually randomize augmentation pool.
#
train_loader.dataset.transform_object._BatchAdapter__augmenter_pool.randomize()
data, target = data.numpy(), target.numpy()
train_loader.dataset.running_idx += train_loader.batch_size
if train_loader.dataset.running_idx >= train_loader.dataset.buffer_size:
cmd.logger.info('Running index of train loader is: {}, updating buffer...'.format(train_loader.dataset.running_idx))
train_loader.dataset._load_next()
output = model.update(data, target)
training_stats.update(output)
if i % 100 == 0:
cmd.logger.info('Epoch: {}, training step: {} / {}'.format(epoch, i, train_loader.batches_per_epoch))
if i >= train_loader.batches_per_epoch - 1:
break
training_steps = train_loader.batches_per_epoch
training_stats = {k: v / training_steps for k, v, in training_stats.items()}
# Report on epoch stats.
#
info_str = ', '.join(['{}: {:.3f}'.format(k, training_stats[k]) for k in model.metricnames()])
cmd.logger.info('Training: ' + info_str)
for stat, value in training_stats.items():
experiment.send_metric('train_'+stat, value)
# Validation epoch.
#
validation_stats = Counter()
for i, (data, target) in enumerate(valid_loader):
if isinstance(valid_loader, torch.utils.data.DataLoader):
data, target = data.numpy(), target.numpy()
output = model.validate(data, target)
validation_stats.update(output)
if i % 100 == 0:
cmd.logger.info('Epoch: {}, validation step: {} / {}'.format(epoch, i, valid_loader.batches_per_epoch))
if i >= valid_loader.batches_per_epoch - 1:
break
validation_steps = valid_loader.batches_per_epoch
validation_stats = {k: v / validation_steps for k, v, in validation_stats.items()}
# Report on epoch stats.
#
info_str = ', '.join(['{}: {:.3f}'.format(k, validation_stats[k]) for k in model.metricnames()])
cmd.logger.info('Validation: ' + info_str)
for stat, value in validation_stats.items():
experiment.send_metric('valid_'+stat, value)
# Write output to csv file.
#
csv_output = [training_stats[k] for k in model.metricnames()] + [validation_stats[k] for k in model.metricnames()]
cmd.log_csv_row(csv_output)
# Normalize head report.
#
if config.model['type'] == 'mheads':
for purpose in model.head_report:
total = sum(model.head_report[purpose].values())
model.head_report[purpose] = [model.head_report[purpose][i] / total for i in range(config.model['num heads'])]
# Log head report and reset counter.
#
cmd.log_normalized_head_report(model.head_report)
experiment.send_text('training head report', x=epoch, y=str(model.head_report['training']))
experiment.send_text('validation head report', x=epoch, y=str(model.head_report['validation']))
model.reset_head_report()
# Update state based on validation stats for checkpointing and early stopping.
#
_ = state.update_state(validation_stats)
# Increment epoch counter.
#
state.increment_epoch()
# Save state and model.
#
if state.improved():
cmd.save(model, state, best=True)
cmd.save(model, state, best=False)
# Report best validation metric and index of the epoch.
#
cmd.logger.info('Best validation {metric} was {value} at epoch {index}'.format(metric=config.training['metric name'],
value=state['best_plateau_metric'],
index=state['best_plateau_metric_epoch']))
experiment.stop()
# In[8]:
#code
print("Coucou")
# In[9]:
nb_name = "00_core.ipynb"
# In[11]:
# Neptune login
from neptune.sessions import Session
from fast_neptune.core import fast_experiment
import getpass
api_token = getpass.getpass("Please enter your NeptuneML API token : ")
session = Session(api_token=api_token)
project = session.get_project(project_qualified_name='danywin/fast-neptune')
# In[13]:
globs = globals()
# In[ ]:
with fast_experiment(project, nb_name, globs) as exp:
pass