def test_send_values_from_multiple_channels(self):
# given
numeric_values = [
ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3)
]
text_values = [
ChannelValue(x=i, y="text", ts=self._TS + i) for i in range(0, 3)
]
image_values = [
ChannelValue(x=i,
y={'image_value': {
'data': "base64Image=="
}},
ts=self._TS + i) for i in range(0, 3)
]
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in numeric_values:
channels_values_sender.send(self._NUMERIC_CHANNEL.name,
self._NUMERIC_CHANNEL.channelType,
channel_value)
for channel_value in text_values:
channels_values_sender.send(self._TEXT_CHANNEL.name,
self._TEXT_CHANNEL.channelType,
channel_value)
for channel_value in image_values:
channels_values_sender.send(self._IMAGE_CHANNEL.name,
self._IMAGE_CHANNEL.channelType,
channel_value)
# and
channels_values_sender.join()
# then
# pylint: disable=protected-access
(args, _) = self._EXPERIMENT._send_channels_values.call_args
self.assertEqual(len(args), 1)
self.assertEqual(
sorted(args[0]),
sorted([
ChannelIdWithValues(channel_id=self._NUMERIC_CHANNEL.id,
channel_values=numeric_values),
ChannelIdWithValues(channel_id=self._TEXT_CHANNEL.id,
channel_values=text_values),
ChannelIdWithValues(channel_id=self._IMAGE_CHANNEL.id,
channel_values=image_values)
]))
def test_send_when_waiting_for_next_value_timed_out(self):
# given
numeric_values = [
ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3)
]
# and
semaphore = threading.Semaphore(0)
# pylint: disable=protected-access
self._EXPERIMENT._send_channels_values.side_effect = lambda _: semaphore.release(
)
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in numeric_values:
channels_values_sender.send(self._NUMERIC_CHANNEL.name,
self._NUMERIC_CHANNEL.channelType,
channel_value)
# then
# pylint: disable=protected-access
semaphore.acquire()
self._EXPERIMENT._send_channels_values.assert_called_with([
ChannelIdWithValues(channel_id=self._NUMERIC_CHANNEL.id,
channel_values=numeric_values)
])
# and
self._EXPERIMENT._send_channels_values.reset_mock()
channels_values_sender.join()
# and
self._EXPERIMENT._send_channels_values.assert_not_called()
def _send_values(self, queued_channels_values):
def get_channel_id(value):
return value.channel_id
queued_grouped_by_channel = {channel_id: list(values)
for channel_id, values
in groupby(sorted(queued_channels_values, key=get_channel_id),
get_channel_id)}
channels_with_values = []
for channel_id in queued_grouped_by_channel:
channel_values = []
for queued_value in queued_grouped_by_channel[channel_id]:
channel_values.append(ChannelValue(ts=queued_value.channel_value.ts,
x=queued_value.channel_value.x,
y=queued_value.channel_value.y))
channels_with_values.append(ChannelIdWithValues(channel_id, channel_values))
try:
# pylint:disable=protected-access
self._experiment._send_channels_values(channels_with_values)
except HTTPUnprocessableEntity as e:
message = "Maximum storage limit reached"
try:
message = e.response.json()["message"]
finally:
_logger.warning('Failed to send channel value: %s', message)
except (NeptuneException, IOError):
_logger.exception('Failed to send channel value.')
def test_send_values_on_join(self):
# given
channel_value = ChannelValue(x=1, y="value", ts=self._TS)
# and
channels_values_sender = ChannelsValuesSender(experiment=self._EXPERIMENT)
# when
channels_values_sender.send(
self._TEXT_CHANNEL.name, self._TEXT_CHANNEL.channelType, channel_value
)
# and
channels_values_sender.join()
# then
# pylint: disable=protected-access
self._EXPERIMENT._send_channels_values.assert_called_with(
[
ChannelIdWithValues(
channel_id=self._TEXT_CHANNEL.id,
channel_name=self._TEXT_CHANNEL.name,
channel_type=self._TEXT_CHANNEL.channelType,
channel_namespace=ChannelNamespace.USER,
channel_values=[channel_value],
)
]
)
def test_send_image(self, ChannelsValuesSender, content):
# given
channels_values_sender = ChannelsValuesSender.return_value
experiment = Experiment(mock.MagicMock(), a_project(),
an_experiment_id(), a_uuid_string())
image_value = dict(
name=a_string(),
description=a_string(),
data=base64.b64encode(content()).decode("utf-8"),
)
channel_value = ChannelValue(x=random.randint(0, 100),
y=dict(image_value=image_value),
ts=time.time())
# when
experiment.send_image(
"errors",
channel_value.x,
"/tmp/img.png",
image_value["name"],
image_value["description"],
channel_value.ts,
)
# then
channels_values_sender.send.assert_called_with("errors",
ChannelType.IMAGE.value,
channel_value)
def test_send_when_waiting_for_next_value_timed_out(self):
# given
numeric_values = [
ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3)
]
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in numeric_values:
channels_values_sender.send(self._NUMERIC_CHANNEL.name,
self._NUMERIC_CHANNEL.channelType,
channel_value)
# and
time.sleep(self.__TIMEOUT * 2)
# then
# pylint: disable=protected-access
self._EXPERIMENT._send_channels_values.assert_called_with([
ChannelIdWithValues(channel_id=self._NUMERIC_CHANNEL.id,
channel_values=numeric_values)
])
# and
self._EXPERIMENT._send_channels_values.reset_mock()
channels_values_sender.join()
# and
self._EXPERIMENT._send_channels_values.assert_not_called()
def _send_values(self, queued_channels_values):
channel_key = lambda value: (value.channel_name, value.channel_type)
queued_grouped_by_channel = {
channel: list(values)
for channel, values in groupby(
sorted(queued_channels_values, key=channel_key), channel_key)
}
channels_with_values = []
for (channel_name, channel_type) in queued_grouped_by_channel:
# pylint: disable=protected-access
channel = self._experiment._get_channel(channel_name, channel_type)
last_x = channel.x if channel.x else 0
channel_values = []
for queued_value in queued_grouped_by_channel[(channel_name,
channel_type)]:
x = queued_value.channel_value.x if queued_value.channel_value.x is not None else last_x + 1
channel_values.append(
ChannelValue(ts=queued_value.channel_value.ts,
x=x,
y=queued_value.channel_value.y))
last_x = x
channels_with_values.append(
ChannelIdWithValues(channel.id, channel_values))
# pylint: disable=protected-access
try:
self._experiment._send_channels_values(channels_with_values)
except (NeptuneApiException, IOError):
pass
def log_text(self, log_name, x, y=None, timestamp=None):
"""Log text data in Neptune
| If a log with provided ``log_name`` does not exist, it is created automatically.
| If log exists (determined by ``log_name``), then new value is appended to it.
Args:
log_name (:obj:`str`): The name of log, i.e. `mse`, `my_text_data`, `timing_info`.
x (:obj:`double` or :obj:`str`): Depending, whether ``y`` parameter is passed:
* ``y`` not passed: The value of the log (data-point). Must be ``str``.
* ``y`` passed: Index of log entry being appended. Must be strictly increasing.
y (:obj:`str`, optional, default is ``None``): The value of the log (data-point).
timestamp (:obj:`time`, optional, default is ``None``):
Timestamp to be associated with log entry. Must be Unix time.
If ``None`` is passed, `time.time() <https://docs.python.org/3.6/library/time.html#time.time>`_
(Python 3.6 example) is invoked to obtain timestamp.
Example:
Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`:
.. code:: python3
# common case, where log name and data are passed
neptune.log_text('my_text_data', str(data_item))
# log_name, x and timestamp are passed
neptune.log_text(log_name='logging_losses_as_text',
x=str(val_loss),
timestamp=1560430912)
Note:
For efficiency, logs are uploaded in batches via a queue.
Hence, if you log a lot of data, you may experience slight delays in Neptune web application.
Note:
Passing ``x`` coordinate as NaN or +/-inf causes this log entry to be ignored.
Warning is printed to ``stdout``.
"""
x, y = self._get_valid_x_y(x, y)
if x is not None and is_nan_or_inf(x):
x = None
if not isinstance(y, six.string_types):
raise InvalidChannelValue(expected_type="str",
actual_type=type(y).__name__)
if x is not None and is_nan_or_inf(x):
_logger.warning(
"Invalid metric x-coordinate: %s for channel %s. "
"Metrics with nan or +/-inf x-coordinates will not be sent to server",
x,
log_name,
)
else:
value = ChannelValue(x, dict(text_value=y), timestamp)
self._channels_values_sender.send(log_name, ChannelType.TEXT.value,
value)
def send_metric(self, channel_name, x, y=None, timestamp=None):
x, y = self._get_valid_x_y(x, y)
if not is_float(y):
raise InvalidChannelValue(expected_type='float', actual_type=type(y).__name__)
value = ChannelValue(x, dict(numeric_value=y), timestamp)
self._channels_values_sender.send(channel_name, 'numeric', value)
def send_text(self, channel_name, x, y=None, timestamp=None):
x, y = self._get_valid_x_y(x, y)
if not isinstance(y, six.string_types):
raise InvalidChannelValue(expected_type='str', actual_type=type(y).__name__)
value = ChannelValue(x, dict(text_value=y), timestamp)
self._channels_values_sender.send(channel_name, 'text', value)
def send_image(self, channel_name, x, y=None, name=None, description=None, timestamp=None):
x, y = self._get_valid_x_y(x, y)
input_image = dict(
name=name,
description=description,
data=base64.b64encode(get_image_content(y)).decode('utf-8')
)
value = ChannelValue(x, dict(image_value=input_image), timestamp)
self._channels_values_sender.send(channel_name, 'image', value)
def _test_send_channel_values(
self,
channel_y_elements: List[tuple],
expected_operation: str,
channel_type: ChannelType,
):
# given prepared `ChannelIdWithValues`
channel_id = "channel_id"
channel_name = "channel_name"
now_ms = int(time.time() * 1000)
channel_with_values = ChannelIdWithValues(
channel_id=channel_id,
channel_name=channel_name,
channel_type=channel_type.value,
channel_namespace=ChannelNamespace.USER,
channel_values=[
ChannelValue(x=None,
y={channel_y_key: channel_y_value},
ts=None)
for channel_y_key, channel_y_value in channel_y_elements
],
)
# invoke send_channels_values
self.leaderboard.send_channels_values(self.exp_mock,
[channel_with_values])
# expect `executeOperations` was called once with properly prepared kwargs
expected_call_args = {
"experimentId":
"00000000-0000-0000-0000-000000000000",
"operations": [{
"path": f"logs/{channel_name}",
expected_operation: {
"entries": [{
"value": channel_y_value,
"step": None,
"timestampMilliseconds": now_ms,
} for _, channel_y_value in channel_y_elements]
},
}],
}
# pylint:disable=protected-access
execute_operations = (
self.leaderboard.leaderboard_swagger_client.api.executeOperations)
self.assertEqual(len(execute_operations.call_args_list), 1)
self.assertDictEqual(execute_operations.call_args_list[0][1],
expected_call_args)
def test_send_images_in_smaller_batches(self):
# and
value = "base64Image=="
channels_values = [
ChannelValue(
x=i,
y={
'image_value': {
'data':
value + value * int(self._IMAGES_BATCH_IMAGE_SIZE /
(len(value)))
}
},
ts=self._TS + i) for i in range(0, self._IMAGES_BATCH_SIZE * 3)
]
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in channels_values:
channels_values_sender.send(self._IMAGE_CHANNEL.name,
self._IMAGE_CHANNEL.channelType,
channel_value)
# and
channels_values_sender.join()
# then
# pylint: disable=protected-access
self.assertEqual(self._EXPERIMENT._send_channels_values.mock_calls, [
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._IMAGE_CHANNEL.id,
channel_values=channels_values[0:self._IMAGES_BATCH_SIZE])
]),
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._IMAGE_CHANNEL.id,
channel_values=channels_values[self._IMAGES_BATCH_SIZE:self
._IMAGES_BATCH_SIZE * 2])
]),
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._IMAGE_CHANNEL.id,
channel_values=channels_values[self._IMAGES_BATCH_SIZE *
2:])
])
])
def test_send_text(self, ChannelsValuesSender):
# given
channels_values_sender = ChannelsValuesSender.return_value
experiment = Experiment(mock.MagicMock(), a_project(),
an_experiment_id(), a_uuid_string())
channel_value = ChannelValue(x=random.randint(0, 100),
y=dict(text_value=a_string()),
ts=time.time())
# when
experiment.send_text("stdout", channel_value.x,
channel_value.y["text_value"], channel_value.ts)
# then
channels_values_sender.send.assert_called_with("stdout",
ChannelType.TEXT.value,
channel_value)
def write(self, data):
if self._data is None:
self._data = data
else:
self._data += data
lines = self.__SPLIT_PATTERN.split(self._data)
for line in lines[:-1]:
value = ChannelValue(x=self._x_offset.next(),
y=dict(text_value=str(line)),
ts=None)
# pylint: disable=protected-access
self._experiment._channels_values_sender.send(
channel_name=self._channel_name,
channel_type=ChannelType.TEXT.value,
channel_value=value,
channel_namespace=self._channel_namespace)
self._data = lines[-1]
def test_send_metric(self, ChannelsValuesSender):
# given
channels_values_sender = ChannelsValuesSender.return_value
experiment = Experiment(mock.MagicMock(), mock.MagicMock(),
an_experiment_id(), a_uuid_string())
channel_value = ChannelValue(
x=random.randint(0, 100),
y=dict(numeric_value=random.randint(0, 100)),
ts=time.time())
# when
experiment.send_metric('loss', channel_value.x,
channel_value.y['numeric_value'],
channel_value.ts)
# then
channels_values_sender.send.assert_called_with(
'loss', ChannelType.NUMERIC.value, channel_value)
def write(self, data):
if self._data is None:
self._data = data
else:
self._data += data
lines = self.__SPLIT_PATTERN.split(self._data)
for line in lines[:-1]:
value = ChannelValue(
x=(datetime.now(tz=self._time_started.tzinfo) -
self._time_started).total_seconds() * 1000,
y=dict(text_value=str(line)),
ts=None)
# pylint: disable=protected-access
self._experiment._channels_values_sender.send(
channel_name=self._channel_name,
channel_type=ChannelType.TEXT.value,
channel_value=value,
channel_namespace=self._channel_namespace)
self._data = lines[-1]
def test_send_values_in_multiple_batches(self):
# given
channels_values = [
ChannelValue(x=i, y="value{}".format(i), ts=self._TS + i)
for i in range(0, self._BATCH_SIZE * 3)
]
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in channels_values:
channels_values_sender.send(self._TEXT_CHANNEL.name,
self._TEXT_CHANNEL.channelType,
channel_value)
# and
channels_values_sender.join()
# then
# pylint: disable=protected-access
self.assertEqual(self._EXPERIMENT._send_channels_values.mock_calls, [
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._TEXT_CHANNEL.id,
channel_values=channels_values[0:self._BATCH_SIZE])
]),
mock.call._send_channels_values([
ChannelIdWithValues(channel_id=self._TEXT_CHANNEL.id,
channel_values=channels_values[
self._BATCH_SIZE:self._BATCH_SIZE * 2])
]),
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._TEXT_CHANNEL.id,
channel_values=channels_values[self._BATCH_SIZE *
2:self._BATCH_SIZE * 3])
])
])
def log_image(self,
log_name,
x,
y=None,
image_name=None,
description=None,
timestamp=None):
"""Log image data in Neptune
| If a log with provided ``log_name`` does not exist, it is created automatically.
| If log exists (determined by ``log_name``), then new value is appended to it.
Args:
log_name (:obj:`str`): The name of log, i.e. `bboxes`, `visualisations`, `sample_images`.
x (:obj:`double`): Depending, whether ``y`` parameter is passed:
* ``y`` not passed: The value of the log (data-point). See ``y`` parameter.
* ``y`` passed: Index of log entry being appended. Must be strictly increasing.
y (multiple types supported, optional, default is ``None``):
The value of the log (data-point). Can be one of the following types:
* :obj:`PIL image`
`Pillow docs <https://pillow.readthedocs.io/en/latest/reference/Image.html#image-module>`_
* :obj:`matplotlib.figure.Figure`
`Matplotlib 3.1.1 docs <https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.figure.Figure.html>`_
* :obj:`str` - path to image file
* 2-dimensional :obj:`numpy.array` - interpreted as grayscale image
* 3-dimensional :obj:`numpy.array` - behavior depends on last dimension
* if last dimension is 1 - interpreted as grayscale image
* if last dimension is 3 - interpreted as RGB image
* if last dimension is 4 - interpreted as RGBA image
image_name (:obj:`str`, optional, default is ``None``): Image name
description (:obj:`str`, optional, default is ``None``): Image description
timestamp (:obj:`time`, optional, default is ``None``):
Timestamp to be associated with log entry. Must be Unix time.
If ``None`` is passed, `time.time() <https://docs.python.org/3.6/library/time.html#time.time>`_
(Python 3.6 example) is invoked to obtain timestamp.
Example:
Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`:
.. code:: python3
# path to image file
experiment.log_image('bbox_images', 'pictures/image.png')
experiment.log_image('bbox_images', x=5, 'pictures/image.png')
experiment.log_image('bbox_images', 'pictures/image.png', image_name='difficult_case')
# PIL image
img = PIL.Image.new('RGB', (60, 30), color = 'red')
experiment.log_image('fig', img)
# 2d numpy array
array = numpy.random.rand(300, 200)*255
experiment.log_image('fig', array)
# 3d grayscale array
array = numpy.random.rand(300, 200, 1)*255
experiment.log_image('fig', array)
# 3d RGB array
array = numpy.random.rand(300, 200, 3)*255
experiment.log_image('fig', array)
# 3d RGBA array
array = numpy.random.rand(300, 200, 4)*255
experiment.log_image('fig', array)
# matplotlib figure example 1
from matplotlib import pyplot
pyplot.plot([1, 2, 3, 4])
pyplot.ylabel('some numbers')
experiment.log_image('plots', plt.gcf())
# matplotlib figure example 2
from matplotlib import pyplot
import numpy
numpy.random.seed(19680801)
data = numpy.random.randn(2, 100)
figure, axs = pyplot.subplots(2, 2, figsize=(5, 5))
axs[0, 0].hist(data[0])
axs[1, 0].scatter(data[0], data[1])
axs[0, 1].plot(data[0], data[1])
axs[1, 1].hist2d(data[0], data[1])
experiment.log_image('diagrams', figure)
Note:
For efficiency, logs are uploaded in batches via a queue.
Hence, if you log a lot of data, you may experience slight delays in Neptune web application.
Note:
Passing ``x`` coordinate as NaN or +/-inf causes this log entry to be ignored.
Warning is printed to ``stdout``.
Warning:
Only images up to 2MB are supported. Larger files will not be logged to Neptune.
"""
x, y = self._get_valid_x_y(x, y)
if x is not None and is_nan_or_inf(x):
x = None
image_content = get_image_content(y)
if len(image_content) > self.IMAGE_SIZE_LIMIT:
_logger.warning(
'Your image is larger than 2MB. Neptune supports logging images smaller than 2MB. '
'Resize or increase compression of this image')
image_content = None
input_image = dict(name=image_name, description=description)
if image_content:
input_image['data'] = base64.b64encode(image_content).decode(
'utf-8')
if x is not None and is_nan_or_inf(x):
_logger.warning(
'Invalid metric x-coordinate: %s for channel %s. '
'Metrics with nan or +/-inf x-coordinates will not be sent to server',
x, log_name)
else:
value = ChannelValue(x, dict(image_value=input_image), timestamp)
self._channels_values_sender.send(log_name,
ChannelType.IMAGE.value, value)
def log_metric(self, log_name, x, y=None, timestamp=None):
"""Log metrics (numeric values) in Neptune
| If a log with provided ``log_name`` does not exist, it is created automatically.
| If log exists (determined by ``log_name``), then new value is appended to it.
Args:
log_name (:obj:`str`): The name of log, i.e. `mse`, `loss`, `accuracy`.
x (:obj:`double`): Depending, whether ``y`` parameter is passed:
* ``y`` not passed: The value of the log (data-point).
* ``y`` passed: Index of log entry being appended. Must be strictly increasing.
y (:obj:`double`, optional, default is ``None``): The value of the log (data-point).
timestamp (:obj:`time`, optional, default is ``None``):
Timestamp to be associated with log entry. Must be Unix time.
If ``None`` is passed, `time.time() <https://docs.python.org/3.6/library/time.html#time.time>`_
(Python 3.6 example) is invoked to obtain timestamp.
Example:
Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment` and
'accuracy' log does not exists:
.. code:: python3
# Both calls below have the same effect
# Common invocation, providing log name and value
experiment.log_metric('accuracy', 0.5)
experiment.log_metric('accuracy', 0.65)
experiment.log_metric('accuracy', 0.8)
# Providing both x and y params
experiment.log_metric('accuracy', 0, 0.5)
experiment.log_metric('accuracy', 1, 0.65)
experiment.log_metric('accuracy', 2, 0.8)
Note:
For efficiency, logs are uploaded in batches via a queue.
Hence, if you log a lot of data, you may experience slight delays in Neptune web application.
Note:
Passing either ``x`` or ``y`` coordinate as NaN or +/-inf causes this log entry to be ignored.
Warning is printed to ``stdout``.
"""
x, y = self._get_valid_x_y(x, y)
if not is_float(y):
raise InvalidChannelValue(expected_type='float',
actual_type=type(y).__name__)
if is_nan_or_inf(y):
_logger.warning(
'Invalid metric value: %s for channel %s. '
'Metrics with nan or +/-inf values will not be sent to server',
y, log_name)
elif x is not None and is_nan_or_inf(x):
_logger.warning(
'Invalid metric x-coordinate: %s for channel %s. '
'Metrics with nan or +/-inf x-coordinates will not be sent to server',
x, log_name)
else:
value = ChannelValue(x, dict(numeric_value=y), timestamp)
self._channels_values_sender.send(log_name,
ChannelType.NUMERIC.value, value)
