def test_send_values_from_multiple_channels(self): # given numeric_values = [ ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3) ] text_values = [ ChannelValue(x=i, y="text", ts=self._TS + i) for i in range(0, 3) ] image_values = [ ChannelValue(x=i, y={'image_value': { 'data': "base64Image==" }}, ts=self._TS + i) for i in range(0, 3) ] # and channels_values_sender = ChannelsValuesSender( experiment=self._EXPERIMENT) # when for channel_value in numeric_values: channels_values_sender.send(self._NUMERIC_CHANNEL.name, self._NUMERIC_CHANNEL.channelType, channel_value) for channel_value in text_values: channels_values_sender.send(self._TEXT_CHANNEL.name, self._TEXT_CHANNEL.channelType, channel_value) for channel_value in image_values: channels_values_sender.send(self._IMAGE_CHANNEL.name, self._IMAGE_CHANNEL.channelType, channel_value) # and channels_values_sender.join() # then # pylint: disable=protected-access (args, _) = self._EXPERIMENT._send_channels_values.call_args self.assertEqual(len(args), 1) self.assertEqual( sorted(args[0]), sorted([ ChannelIdWithValues(channel_id=self._NUMERIC_CHANNEL.id, channel_values=numeric_values), ChannelIdWithValues(channel_id=self._TEXT_CHANNEL.id, channel_values=text_values), ChannelIdWithValues(channel_id=self._IMAGE_CHANNEL.id, channel_values=image_values) ]))
def test_send_when_waiting_for_next_value_timed_out(self): # given numeric_values = [ ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3) ] # and semaphore = threading.Semaphore(0) # pylint: disable=protected-access self._EXPERIMENT._send_channels_values.side_effect = lambda _: semaphore.release( ) # and channels_values_sender = ChannelsValuesSender( experiment=self._EXPERIMENT) # when for channel_value in numeric_values: channels_values_sender.send(self._NUMERIC_CHANNEL.name, self._NUMERIC_CHANNEL.channelType, channel_value) # then # pylint: disable=protected-access semaphore.acquire() self._EXPERIMENT._send_channels_values.assert_called_with([ ChannelIdWithValues(channel_id=self._NUMERIC_CHANNEL.id, channel_values=numeric_values) ]) # and self._EXPERIMENT._send_channels_values.reset_mock() channels_values_sender.join() # and self._EXPERIMENT._send_channels_values.assert_not_called()
def _send_values(self, queued_channels_values): def get_channel_id(value): return value.channel_id queued_grouped_by_channel = {channel_id: list(values) for channel_id, values in groupby(sorted(queued_channels_values, key=get_channel_id), get_channel_id)} channels_with_values = [] for channel_id in queued_grouped_by_channel: channel_values = [] for queued_value in queued_grouped_by_channel[channel_id]: channel_values.append(ChannelValue(ts=queued_value.channel_value.ts, x=queued_value.channel_value.x, y=queued_value.channel_value.y)) channels_with_values.append(ChannelIdWithValues(channel_id, channel_values)) try: # pylint:disable=protected-access self._experiment._send_channels_values(channels_with_values) except HTTPUnprocessableEntity as e: message = "Maximum storage limit reached" try: message = e.response.json()["message"] finally: _logger.warning('Failed to send channel value: %s', message) except (NeptuneException, IOError): _logger.exception('Failed to send channel value.')
def test_send_values_on_join(self): # given channel_value = ChannelValue(x=1, y="value", ts=self._TS) # and channels_values_sender = ChannelsValuesSender(experiment=self._EXPERIMENT) # when channels_values_sender.send( self._TEXT_CHANNEL.name, self._TEXT_CHANNEL.channelType, channel_value ) # and channels_values_sender.join() # then # pylint: disable=protected-access self._EXPERIMENT._send_channels_values.assert_called_with( [ ChannelIdWithValues( channel_id=self._TEXT_CHANNEL.id, channel_name=self._TEXT_CHANNEL.name, channel_type=self._TEXT_CHANNEL.channelType, channel_namespace=ChannelNamespace.USER, channel_values=[channel_value], ) ] )
def test_send_image(self, ChannelsValuesSender, content): # given channels_values_sender = ChannelsValuesSender.return_value experiment = Experiment(mock.MagicMock(), a_project(), an_experiment_id(), a_uuid_string()) image_value = dict( name=a_string(), description=a_string(), data=base64.b64encode(content()).decode("utf-8"), ) channel_value = ChannelValue(x=random.randint(0, 100), y=dict(image_value=image_value), ts=time.time()) # when experiment.send_image( "errors", channel_value.x, "/tmp/img.png", image_value["name"], image_value["description"], channel_value.ts, ) # then channels_values_sender.send.assert_called_with("errors", ChannelType.IMAGE.value, channel_value)
def test_send_when_waiting_for_next_value_timed_out(self): # given numeric_values = [ ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3) ] # and channels_values_sender = ChannelsValuesSender( experiment=self._EXPERIMENT) # when for channel_value in numeric_values: channels_values_sender.send(self._NUMERIC_CHANNEL.name, self._NUMERIC_CHANNEL.channelType, channel_value) # and time.sleep(self.__TIMEOUT * 2) # then # pylint: disable=protected-access self._EXPERIMENT._send_channels_values.assert_called_with([ ChannelIdWithValues(channel_id=self._NUMERIC_CHANNEL.id, channel_values=numeric_values) ]) # and self._EXPERIMENT._send_channels_values.reset_mock() channels_values_sender.join() # and self._EXPERIMENT._send_channels_values.assert_not_called()
def _send_values(self, queued_channels_values): channel_key = lambda value: (value.channel_name, value.channel_type) queued_grouped_by_channel = { channel: list(values) for channel, values in groupby( sorted(queued_channels_values, key=channel_key), channel_key) } channels_with_values = [] for (channel_name, channel_type) in queued_grouped_by_channel: # pylint: disable=protected-access channel = self._experiment._get_channel(channel_name, channel_type) last_x = channel.x if channel.x else 0 channel_values = [] for queued_value in queued_grouped_by_channel[(channel_name, channel_type)]: x = queued_value.channel_value.x if queued_value.channel_value.x is not None else last_x + 1 channel_values.append( ChannelValue(ts=queued_value.channel_value.ts, x=x, y=queued_value.channel_value.y)) last_x = x channels_with_values.append( ChannelIdWithValues(channel.id, channel_values)) # pylint: disable=protected-access try: self._experiment._send_channels_values(channels_with_values) except (NeptuneApiException, IOError): pass
def log_text(self, log_name, x, y=None, timestamp=None): """Log text data in Neptune | If a log with provided ``log_name`` does not exist, it is created automatically. | If log exists (determined by ``log_name``), then new value is appended to it. Args: log_name (:obj:`str`): The name of log, i.e. `mse`, `my_text_data`, `timing_info`. x (:obj:`double` or :obj:`str`): Depending, whether ``y`` parameter is passed: * ``y`` not passed: The value of the log (data-point). Must be ``str``. * ``y`` passed: Index of log entry being appended. Must be strictly increasing. y (:obj:`str`, optional, default is ``None``): The value of the log (data-point). timestamp (:obj:`time`, optional, default is ``None``): Timestamp to be associated with log entry. Must be Unix time. If ``None`` is passed, `time.time() <https://docs.python.org/3.6/library/time.html#time.time>`_ (Python 3.6 example) is invoked to obtain timestamp. Example: Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: .. code:: python3 # common case, where log name and data are passed neptune.log_text('my_text_data', str(data_item)) # log_name, x and timestamp are passed neptune.log_text(log_name='logging_losses_as_text', x=str(val_loss), timestamp=1560430912) Note: For efficiency, logs are uploaded in batches via a queue. Hence, if you log a lot of data, you may experience slight delays in Neptune web application. Note: Passing ``x`` coordinate as NaN or +/-inf causes this log entry to be ignored. Warning is printed to ``stdout``. """ x, y = self._get_valid_x_y(x, y) if x is not None and is_nan_or_inf(x): x = None if not isinstance(y, six.string_types): raise InvalidChannelValue(expected_type="str", actual_type=type(y).__name__) if x is not None and is_nan_or_inf(x): _logger.warning( "Invalid metric x-coordinate: %s for channel %s. " "Metrics with nan or +/-inf x-coordinates will not be sent to server", x, log_name, ) else: value = ChannelValue(x, dict(text_value=y), timestamp) self._channels_values_sender.send(log_name, ChannelType.TEXT.value, value)
def send_metric(self, channel_name, x, y=None, timestamp=None): x, y = self._get_valid_x_y(x, y) if not is_float(y): raise InvalidChannelValue(expected_type='float', actual_type=type(y).__name__) value = ChannelValue(x, dict(numeric_value=y), timestamp) self._channels_values_sender.send(channel_name, 'numeric', value)
def send_text(self, channel_name, x, y=None, timestamp=None): x, y = self._get_valid_x_y(x, y) if not isinstance(y, six.string_types): raise InvalidChannelValue(expected_type='str', actual_type=type(y).__name__) value = ChannelValue(x, dict(text_value=y), timestamp) self._channels_values_sender.send(channel_name, 'text', value)
def send_image(self, channel_name, x, y=None, name=None, description=None, timestamp=None): x, y = self._get_valid_x_y(x, y) input_image = dict( name=name, description=description, data=base64.b64encode(get_image_content(y)).decode('utf-8') ) value = ChannelValue(x, dict(image_value=input_image), timestamp) self._channels_values_sender.send(channel_name, 'image', value)
def _test_send_channel_values( self, channel_y_elements: List[tuple], expected_operation: str, channel_type: ChannelType, ): # given prepared `ChannelIdWithValues` channel_id = "channel_id" channel_name = "channel_name" now_ms = int(time.time() * 1000) channel_with_values = ChannelIdWithValues( channel_id=channel_id, channel_name=channel_name, channel_type=channel_type.value, channel_namespace=ChannelNamespace.USER, channel_values=[ ChannelValue(x=None, y={channel_y_key: channel_y_value}, ts=None) for channel_y_key, channel_y_value in channel_y_elements ], ) # invoke send_channels_values self.leaderboard.send_channels_values(self.exp_mock, [channel_with_values]) # expect `executeOperations` was called once with properly prepared kwargs expected_call_args = { "experimentId": "00000000-0000-0000-0000-000000000000", "operations": [{ "path": f"logs/{channel_name}", expected_operation: { "entries": [{ "value": channel_y_value, "step": None, "timestampMilliseconds": now_ms, } for _, channel_y_value in channel_y_elements] }, }], } # pylint:disable=protected-access execute_operations = ( self.leaderboard.leaderboard_swagger_client.api.executeOperations) self.assertEqual(len(execute_operations.call_args_list), 1) self.assertDictEqual(execute_operations.call_args_list[0][1], expected_call_args)
def test_send_images_in_smaller_batches(self): # and value = "base64Image==" channels_values = [ ChannelValue( x=i, y={ 'image_value': { 'data': value + value * int(self._IMAGES_BATCH_IMAGE_SIZE / (len(value))) } }, ts=self._TS + i) for i in range(0, self._IMAGES_BATCH_SIZE * 3) ] # and channels_values_sender = ChannelsValuesSender( experiment=self._EXPERIMENT) # when for channel_value in channels_values: channels_values_sender.send(self._IMAGE_CHANNEL.name, self._IMAGE_CHANNEL.channelType, channel_value) # and channels_values_sender.join() # then # pylint: disable=protected-access self.assertEqual(self._EXPERIMENT._send_channels_values.mock_calls, [ mock.call._send_channels_values([ ChannelIdWithValues( channel_id=self._IMAGE_CHANNEL.id, channel_values=channels_values[0:self._IMAGES_BATCH_SIZE]) ]), mock.call._send_channels_values([ ChannelIdWithValues( channel_id=self._IMAGE_CHANNEL.id, channel_values=channels_values[self._IMAGES_BATCH_SIZE:self ._IMAGES_BATCH_SIZE * 2]) ]), mock.call._send_channels_values([ ChannelIdWithValues( channel_id=self._IMAGE_CHANNEL.id, channel_values=channels_values[self._IMAGES_BATCH_SIZE * 2:]) ]) ])
def test_send_text(self, ChannelsValuesSender): # given channels_values_sender = ChannelsValuesSender.return_value experiment = Experiment(mock.MagicMock(), a_project(), an_experiment_id(), a_uuid_string()) channel_value = ChannelValue(x=random.randint(0, 100), y=dict(text_value=a_string()), ts=time.time()) # when experiment.send_text("stdout", channel_value.x, channel_value.y["text_value"], channel_value.ts) # then channels_values_sender.send.assert_called_with("stdout", ChannelType.TEXT.value, channel_value)
def write(self, data): if self._data is None: self._data = data else: self._data += data lines = self.__SPLIT_PATTERN.split(self._data) for line in lines[:-1]: value = ChannelValue(x=self._x_offset.next(), y=dict(text_value=str(line)), ts=None) # pylint: disable=protected-access self._experiment._channels_values_sender.send( channel_name=self._channel_name, channel_type=ChannelType.TEXT.value, channel_value=value, channel_namespace=self._channel_namespace) self._data = lines[-1]
def test_send_metric(self, ChannelsValuesSender): # given channels_values_sender = ChannelsValuesSender.return_value experiment = Experiment(mock.MagicMock(), mock.MagicMock(), an_experiment_id(), a_uuid_string()) channel_value = ChannelValue( x=random.randint(0, 100), y=dict(numeric_value=random.randint(0, 100)), ts=time.time()) # when experiment.send_metric('loss', channel_value.x, channel_value.y['numeric_value'], channel_value.ts) # then channels_values_sender.send.assert_called_with( 'loss', ChannelType.NUMERIC.value, channel_value)
def write(self, data): if self._data is None: self._data = data else: self._data += data lines = self.__SPLIT_PATTERN.split(self._data) for line in lines[:-1]: value = ChannelValue( x=(datetime.now(tz=self._time_started.tzinfo) - self._time_started).total_seconds() * 1000, y=dict(text_value=str(line)), ts=None) # pylint: disable=protected-access self._experiment._channels_values_sender.send( channel_name=self._channel_name, channel_type=ChannelType.TEXT.value, channel_value=value, channel_namespace=self._channel_namespace) self._data = lines[-1]
def test_send_values_in_multiple_batches(self): # given channels_values = [ ChannelValue(x=i, y="value{}".format(i), ts=self._TS + i) for i in range(0, self._BATCH_SIZE * 3) ] # and channels_values_sender = ChannelsValuesSender( experiment=self._EXPERIMENT) # when for channel_value in channels_values: channels_values_sender.send(self._TEXT_CHANNEL.name, self._TEXT_CHANNEL.channelType, channel_value) # and channels_values_sender.join() # then # pylint: disable=protected-access self.assertEqual(self._EXPERIMENT._send_channels_values.mock_calls, [ mock.call._send_channels_values([ ChannelIdWithValues( channel_id=self._TEXT_CHANNEL.id, channel_values=channels_values[0:self._BATCH_SIZE]) ]), mock.call._send_channels_values([ ChannelIdWithValues(channel_id=self._TEXT_CHANNEL.id, channel_values=channels_values[ self._BATCH_SIZE:self._BATCH_SIZE * 2]) ]), mock.call._send_channels_values([ ChannelIdWithValues( channel_id=self._TEXT_CHANNEL.id, channel_values=channels_values[self._BATCH_SIZE * 2:self._BATCH_SIZE * 3]) ]) ])
def log_image(self, log_name, x, y=None, image_name=None, description=None, timestamp=None): """Log image data in Neptune | If a log with provided ``log_name`` does not exist, it is created automatically. | If log exists (determined by ``log_name``), then new value is appended to it. Args: log_name (:obj:`str`): The name of log, i.e. `bboxes`, `visualisations`, `sample_images`. x (:obj:`double`): Depending, whether ``y`` parameter is passed: * ``y`` not passed: The value of the log (data-point). See ``y`` parameter. * ``y`` passed: Index of log entry being appended. Must be strictly increasing. y (multiple types supported, optional, default is ``None``): The value of the log (data-point). Can be one of the following types: * :obj:`PIL image` `Pillow docs <https://pillow.readthedocs.io/en/latest/reference/Image.html#image-module>`_ * :obj:`matplotlib.figure.Figure` `Matplotlib 3.1.1 docs <https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.figure.Figure.html>`_ * :obj:`str` - path to image file * 2-dimensional :obj:`numpy.array` - interpreted as grayscale image * 3-dimensional :obj:`numpy.array` - behavior depends on last dimension * if last dimension is 1 - interpreted as grayscale image * if last dimension is 3 - interpreted as RGB image * if last dimension is 4 - interpreted as RGBA image image_name (:obj:`str`, optional, default is ``None``): Image name description (:obj:`str`, optional, default is ``None``): Image description timestamp (:obj:`time`, optional, default is ``None``): Timestamp to be associated with log entry. Must be Unix time. If ``None`` is passed, `time.time() <https://docs.python.org/3.6/library/time.html#time.time>`_ (Python 3.6 example) is invoked to obtain timestamp. Example: Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: .. code:: python3 # path to image file experiment.log_image('bbox_images', 'pictures/image.png') experiment.log_image('bbox_images', x=5, 'pictures/image.png') experiment.log_image('bbox_images', 'pictures/image.png', image_name='difficult_case') # PIL image img = PIL.Image.new('RGB', (60, 30), color = 'red') experiment.log_image('fig', img) # 2d numpy array array = numpy.random.rand(300, 200)*255 experiment.log_image('fig', array) # 3d grayscale array array = numpy.random.rand(300, 200, 1)*255 experiment.log_image('fig', array) # 3d RGB array array = numpy.random.rand(300, 200, 3)*255 experiment.log_image('fig', array) # 3d RGBA array array = numpy.random.rand(300, 200, 4)*255 experiment.log_image('fig', array) # matplotlib figure example 1 from matplotlib import pyplot pyplot.plot([1, 2, 3, 4]) pyplot.ylabel('some numbers') experiment.log_image('plots', plt.gcf()) # matplotlib figure example 2 from matplotlib import pyplot import numpy numpy.random.seed(19680801) data = numpy.random.randn(2, 100) figure, axs = pyplot.subplots(2, 2, figsize=(5, 5)) axs[0, 0].hist(data[0]) axs[1, 0].scatter(data[0], data[1]) axs[0, 1].plot(data[0], data[1]) axs[1, 1].hist2d(data[0], data[1]) experiment.log_image('diagrams', figure) Note: For efficiency, logs are uploaded in batches via a queue. Hence, if you log a lot of data, you may experience slight delays in Neptune web application. Note: Passing ``x`` coordinate as NaN or +/-inf causes this log entry to be ignored. Warning is printed to ``stdout``. Warning: Only images up to 2MB are supported. Larger files will not be logged to Neptune. """ x, y = self._get_valid_x_y(x, y) if x is not None and is_nan_or_inf(x): x = None image_content = get_image_content(y) if len(image_content) > self.IMAGE_SIZE_LIMIT: _logger.warning( 'Your image is larger than 2MB. Neptune supports logging images smaller than 2MB. ' 'Resize or increase compression of this image') image_content = None input_image = dict(name=image_name, description=description) if image_content: input_image['data'] = base64.b64encode(image_content).decode( 'utf-8') if x is not None and is_nan_or_inf(x): _logger.warning( 'Invalid metric x-coordinate: %s for channel %s. ' 'Metrics with nan or +/-inf x-coordinates will not be sent to server', x, log_name) else: value = ChannelValue(x, dict(image_value=input_image), timestamp) self._channels_values_sender.send(log_name, ChannelType.IMAGE.value, value)
def log_metric(self, log_name, x, y=None, timestamp=None): """Log metrics (numeric values) in Neptune | If a log with provided ``log_name`` does not exist, it is created automatically. | If log exists (determined by ``log_name``), then new value is appended to it. Args: log_name (:obj:`str`): The name of log, i.e. `mse`, `loss`, `accuracy`. x (:obj:`double`): Depending, whether ``y`` parameter is passed: * ``y`` not passed: The value of the log (data-point). * ``y`` passed: Index of log entry being appended. Must be strictly increasing. y (:obj:`double`, optional, default is ``None``): The value of the log (data-point). timestamp (:obj:`time`, optional, default is ``None``): Timestamp to be associated with log entry. Must be Unix time. If ``None`` is passed, `time.time() <https://docs.python.org/3.6/library/time.html#time.time>`_ (Python 3.6 example) is invoked to obtain timestamp. Example: Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment` and 'accuracy' log does not exists: .. code:: python3 # Both calls below have the same effect # Common invocation, providing log name and value experiment.log_metric('accuracy', 0.5) experiment.log_metric('accuracy', 0.65) experiment.log_metric('accuracy', 0.8) # Providing both x and y params experiment.log_metric('accuracy', 0, 0.5) experiment.log_metric('accuracy', 1, 0.65) experiment.log_metric('accuracy', 2, 0.8) Note: For efficiency, logs are uploaded in batches via a queue. Hence, if you log a lot of data, you may experience slight delays in Neptune web application. Note: Passing either ``x`` or ``y`` coordinate as NaN or +/-inf causes this log entry to be ignored. Warning is printed to ``stdout``. """ x, y = self._get_valid_x_y(x, y) if not is_float(y): raise InvalidChannelValue(expected_type='float', actual_type=type(y).__name__) if is_nan_or_inf(y): _logger.warning( 'Invalid metric value: %s for channel %s. ' 'Metrics with nan or +/-inf values will not be sent to server', y, log_name) elif x is not None and is_nan_or_inf(x): _logger.warning( 'Invalid metric x-coordinate: %s for channel %s. ' 'Metrics with nan or +/-inf x-coordinates will not be sent to server', x, log_name) else: value = ChannelValue(x, dict(numeric_value=y), timestamp) self._channels_values_sender.send(log_name, ChannelType.NUMERIC.value, value)