def test_send_image(self, ChannelsValuesSender, content):
# given
channels_values_sender = ChannelsValuesSender.return_value
experiment = Experiment(mock.MagicMock(), a_project(),
an_experiment_id(), a_uuid_string())
image_value = dict(
name=a_string(),
description=a_string(),
data=base64.b64encode(content()).decode("utf-8"),
)
channel_value = ChannelValue(x=random.randint(0, 100),
y=dict(image_value=image_value),
ts=time.time())
# when
experiment.send_image(
"errors",
channel_value.x,
"/tmp/img.png",
image_value["name"],
image_value["description"],
channel_value.ts,
)
# then
channels_values_sender.send.assert_called_with("errors",
ChannelType.IMAGE.value,
channel_value)
def test_delete_artifact(self):
# given
backend = mock.MagicMock()
project = a_project()
experiment = Experiment(
backend,
project,
an_experiment_id(),
a_uuid_string()
)
# and
def build_call(path):
return call(
experiment=experiment,
path=path
)
# when
experiment.delete_artifacts('/an_abs_path_in_exp_output')
experiment.delete_artifacts('/../an_abs_path_in_exp')
experiment.delete_artifacts('/../../an_abs_path_in_prj')
experiment.delete_artifacts('a_path_in_exp_output')
self.assertRaises(ValueError, experiment.delete_artifacts, 'test/../../a_path_outside_exp')
self.assertRaises(ValueError, experiment.delete_artifacts, '../a_path_outside_exp')
self.assertRaises(ValueError, experiment.delete_artifacts, "..")
# then
backend.rm_data.assert_has_calls([
build_call('/an_abs_path_in_exp_output'),
build_call('/../an_abs_path_in_exp'),
build_call('/../../an_abs_path_in_prj'),
build_call('a_path_in_exp_output'),
])
def _convert_to_experiment(self, api_experiment):
return Experiment(client=self,
_id=api_experiment.shortId,
internal_id=api_experiment.id,
project_full_id='{}/{}'.format(
api_experiment.organizationName,
api_experiment.projectName))
def test_get_experiments_with_scalar_params(self):
# given
leaderboard_entries = [MagicMock() for _ in range(0, 2)]
self.backend.get_leaderboard_entries.return_value = leaderboard_entries
# and
params = dict(id=a_string(),
state='succeeded',
owner=a_string(),
tag=a_string(),
min_running_time=randint(1, 100))
# when
experiments = self.project.get_experiments(**params)
# then
expected_params = dict(project=self.project,
ids=[params['id']],
states=[params['state']],
owners=[params['owner']],
tags=[params['tag']],
min_running_time=params['min_running_time'])
self.backend.get_leaderboard_entries.assert_called_once_with(
**expected_params)
# and
expected_experiments = [
Experiment(self.backend, self.project, entry.id, entry.internal_id)
for entry in leaderboard_entries
]
self.assertEqual(expected_experiments, experiments)
def test_get_numeric_channels_values(self):
# when
backend = MagicMock()
backend.get_channel_points_csv.return_value = StringIO(u"\n".join(
["0.3,2.5", "1,2"]))
experiment = MagicMock()
experiment.id = a_string()
experiment.internal_id = a_uuid_string()
experiment.channels = [Munch(id=a_uuid_string(), name="epoch_loss")]
experiment.channelsLastValues = [
Munch(channelName="epoch_loss", x=2.5, y=2)
]
backend.get_experiment.return_value = experiment
# then
experiment = Experiment(
backend=backend,
project=a_project(),
_id=a_string(),
internal_id=a_uuid_string(),
)
result = experiment.get_numeric_channels_values("epoch_loss")
expected_result = pd.DataFrame(
{
"x": [0.3, 1.0],
"epoch_loss": [2.5, 2.0]
}, dtype=float)
expected_result = sort_df_by_columns(expected_result)
result = sort_df_by_columns(result)
assert_frame_equal(expected_result, result)
def test_get_numeric_channels_values(self):
# when
client = MagicMock()
client.get_channel_points_csv.return_value = StringIO(u'\n'.join(['0.3,2.5', '1,2']))
experiment = MagicMock()
experiment.id = a_string()
experiment.internal_id = a_uuid_string()
experiment.channels = [Bunch(id=a_uuid_string(), name='epoch_loss')]
experiment.channelsLastValues = [Bunch(channelName='epoch_loss', x=2.5, y=2)]
client.get_experiment.return_value = experiment
# then
experiment = Experiment(
client=client,
_id=a_string(),
internal_id=a_uuid_string(),
project_full_id="test/sandbox"
)
result = experiment.get_numeric_channels_values('epoch_loss')
expected_result = pd.DataFrame({'x': [0.3, 1.0],
'epoch_loss': [2.5, 2.0]}, dtype=float)
expected_result = sort_df_by_columns(expected_result)
result = sort_df_by_columns(result)
assert_frame_equal(expected_result, result)
def test_send_text(self, ChannelsValuesSender):
# given
channels_values_sender = ChannelsValuesSender.return_value
experiment = Experiment(mock.MagicMock(), a_project(),
an_experiment_id(), a_uuid_string())
channel_value = ChannelValue(x=random.randint(0, 100),
y=dict(text_value=a_string()),
ts=time.time())
# when
experiment.send_text("stdout", channel_value.x,
channel_value.y["text_value"], channel_value.ts)
# then
channels_values_sender.send.assert_called_with("stdout",
ChannelType.TEXT.value,
channel_value)
def _convert_to_experiment(self, api_experiment, project):
# pylint: disable=protected-access
return Experiment(
backend=project._backend,
project=project,
_id=api_experiment.shortId,
internal_id=api_experiment.id,
)
def test_send_metric(self, ChannelsValuesSender):
# given
channels_values_sender = ChannelsValuesSender.return_value
experiment = Experiment(mock.MagicMock(), mock.MagicMock(),
an_experiment_id(), a_uuid_string())
channel_value = ChannelValue(
x=random.randint(0, 100),
y=dict(numeric_value=random.randint(0, 100)),
ts=time.time())
# when
experiment.send_metric('loss', channel_value.x,
channel_value.y['numeric_value'],
channel_value.ts)
# then
channels_values_sender.send.assert_called_with(
'loss', ChannelType.NUMERIC.value, channel_value)
def get_experiments(self,
id=None,
state=None,
owner=None,
tag=None,
min_running_time=None):
"""Retrieve list of experiments matching the specified criteria.
All parameters are optional, each of them specifies a single criterion.
Only experiments matching all of the criteria will be returned.
Args:
id (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``):
| An experiment id like ``'SAN-1'`` or list of ids like ``['SAN-1', 'SAN-2']``.
| Matching any element of the list is sufficient to pass criterion.
state (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``):
| An experiment state like ``'succeeded'`` or list of states like ``['succeeded', 'running']``.
| Possible values: ``'running'``, ``'succeeded'``, ``'failed'``, ``'aborted'``.
| Matching any element of the list is sufficient to pass criterion.
owner (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``):
| *Username* of the experiment owner (User who created experiment is an owner) like ``'josh'``
or list of owners like ``['frederic', 'josh']``.
| Matching any element of the list is sufficient to pass criterion.
tag (:obj:`str` or :obj:`list` of :obj:`str`, optional, default is ``None``):
| An experiment tag like ``'lightGBM'`` or list of tags like ``['pytorch', 'cycleLR']``.
| Only experiments that have all specified tags will match this criterion.
min_running_time (:obj:`int`, optional, default is ``None``):
Minimum running time of an experiment in seconds, like ``2000``.
Returns:
:obj:`list` of :class:`~neptune.experiments.Experiment` objects.
Examples:
.. code:: python3
# Fetch a project
project = session.get_projects('neptune-ai')['neptune-ai/Salt-Detection']
# Get list of experiments
project.get_experiments(state=['aborted'], owner=['neyo'], min_running_time=100000)
# Example output:
# [Experiment(SAL-1609),
# Experiment(SAL-1765),
# Experiment(SAL-1941),
# Experiment(SAL-1960),
# Experiment(SAL-2025)]
"""
leaderboard_entries = self._fetch_leaderboard(id, state, owner, tag,
min_running_time)
return [
Experiment(self._backend, self, entry.id, entry.internal_id)
for entry in leaderboard_entries
]
def get_experiments(self,
id=None,
state=None,
owner=None,
tag=None,
min_running_time=None):
"""Retrieve a list of experiments matching the specified criteria.
All of the parameters of this method are optional, each of them specifies a single criterion.
Only experiments matching all of the criteria will be returned.
If a specific criterion accepts a list (like `state`), experiments matching any element of the list
match this criterion.
Args:
id(list): An ID or list of experiment IDs (rowo.g. 'SAN-1' or ['SAN-1', 'SAN-2'])
state(list): A state or list of experiment states.
E.g. 'succeeded' or ['succeeded', 'preempted'].
Possible states: 'creating', 'waiting', 'initializing', 'running', 'cleaning',
'crashed', 'failed', 'aborted', 'preempted', 'succeeded'
owner(list): The owner or list of owners of the experiments. This parameter expects usernames.
tag(list): A tag or a list of experiment tags. E.g. 'solution-1' or ['solution-1', 'solution-2'].
min_running_time(int): Minimum running time of an experiment in seconds.
Returns:
list: List of `Experiment` objects
Examples:
Instantiate a session.
>>> from neptune.sessions import Session
>>> session = Session()
Fetch a project.
>>> project = session.get_projects('neptune-ml')['neptune-ml/Salt-Detection']
Finally, get a list of experiments that satisfies your criteria:
>>> project.get_experiments(state=['aborted'], owner=['neyo'], min_running_time=100000)
[Experiment(SAL-1609),
Experiment(SAL-1765),
Experiment(SAL-1941),
Experiment(SAL-1960),
Experiment(SAL-2025)]
"""
leaderboard_entries = self._fetch_leaderboard(id, state, owner, tag,
min_running_time)
return [
Experiment(self.client, entry.id, entry.internal_id,
entry.project_full_id) for entry in leaderboard_entries
]
def test_get_experiments_with_no_params(self):
# given
leaderboard_entries = [MagicMock() for _ in range(0, 2)]
self.backend.get_leaderboard_entries.return_value = leaderboard_entries
# when
experiments = self.project.get_experiments()
# then
self.backend.get_leaderboard_entries.assert_called_once_with(
project=self.project,
ids=None,
states=None, owners=None, tags=None,
min_running_time=None)
# and
expected_experiments = [Experiment(self.backend, self.project, entry.id, entry.internal_id)
for entry in leaderboard_entries]
self.assertEqual(expected_experiments, experiments)
def test_append_tags(self):
# given
backend = mock.MagicMock()
experiment = Experiment(backend, a_project(), an_experiment_id(),
a_uuid_string())
# and
def build_call(tags_list):
return call(experiment=experiment,
tags_to_add=tags_list,
tags_to_delete=[])
# when
experiment.append_tag("tag")
experiment.append_tag(["tag1", "tag2", "tag3"])
experiment.append_tag("tag1", "tag2", "tag3")
experiment.append_tags("tag")
experiment.append_tags(["tag1", "tag2", "tag3"])
experiment.append_tags("tag1", "tag2", "tag3")
# then
backend.update_tags.assert_has_calls([
build_call(["tag"]),
build_call(["tag1", "tag2", "tag3"]),
build_call(["tag1", "tag2", "tag3"]),
build_call(["tag"]),
build_call(["tag1", "tag2", "tag3"]),
build_call(["tag1", "tag2", "tag3"]),
])
def _convert_to_experiment(self, api_experiment, project):
return Experiment(backend=self,
project=project,
_id=api_experiment.shortId,
internal_id=api_experiment.id)
def log_neptune_timeline(log:str, exp:Experiment):
exp.log_text('timeline', f'{datetime.now()} - {log}')
def log_evaluation_text(log: str, exp: Experiment):
exp.log_text('evaluation', f'{datetime.now()} - {log}')
logging.info(log)