def test_send_values_from_multiple_channels(self):
# given
numeric_values = [
ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3)
]
text_values = [
ChannelValue(x=i, y="text", ts=self._TS + i) for i in range(0, 3)
]
image_values = [
ChannelValue(x=i,
y={'image_value': {
'data': "base64Image=="
}},
ts=self._TS + i) for i in range(0, 3)
]
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in numeric_values:
channels_values_sender.send(self._NUMERIC_CHANNEL.name,
self._NUMERIC_CHANNEL.channelType,
channel_value)
for channel_value in text_values:
channels_values_sender.send(self._TEXT_CHANNEL.name,
self._TEXT_CHANNEL.channelType,
channel_value)
for channel_value in image_values:
channels_values_sender.send(self._IMAGE_CHANNEL.name,
self._IMAGE_CHANNEL.channelType,
channel_value)
# and
channels_values_sender.join()
# then
# pylint: disable=protected-access
(args, _) = self._EXPERIMENT._send_channels_values.call_args
self.assertEqual(len(args), 1)
self.assertEqual(
sorted(args[0]),
sorted([
ChannelIdWithValues(channel_id=self._NUMERIC_CHANNEL.id,
channel_values=numeric_values),
ChannelIdWithValues(channel_id=self._TEXT_CHANNEL.id,
channel_values=text_values),
ChannelIdWithValues(channel_id=self._IMAGE_CHANNEL.id,
channel_values=image_values)
]))
def test_send_images_in_smaller_batches(self):
# and
value = "base64Image=="
channels_values = [
ChannelValue(
x=i,
y={
'image_value': {
'data':
value + value * int(self._IMAGES_BATCH_IMAGE_SIZE /
(len(value)))
}
},
ts=self._TS + i) for i in range(0, self._IMAGES_BATCH_SIZE * 3)
]
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in channels_values:
channels_values_sender.send(self._IMAGE_CHANNEL.name,
self._IMAGE_CHANNEL.channelType,
channel_value)
# and
channels_values_sender.join()
# then
# pylint: disable=protected-access
self.assertEqual(self._EXPERIMENT._send_channels_values.mock_calls, [
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._IMAGE_CHANNEL.id,
channel_values=channels_values[0:self._IMAGES_BATCH_SIZE])
]),
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._IMAGE_CHANNEL.id,
channel_values=channels_values[self._IMAGES_BATCH_SIZE:self
._IMAGES_BATCH_SIZE * 2])
]),
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._IMAGE_CHANNEL.id,
channel_values=channels_values[self._IMAGES_BATCH_SIZE *
2:])
])
])
def _send_values(self, queued_channels_values):
def get_channel_id(value):
return value.channel_id
queued_grouped_by_channel = {channel_id: list(values)
for channel_id, values
in groupby(sorted(queued_channels_values, key=get_channel_id),
get_channel_id)}
channels_with_values = []
for channel_id in queued_grouped_by_channel:
channel_values = []
for queued_value in queued_grouped_by_channel[channel_id]:
channel_values.append(ChannelValue(ts=queued_value.channel_value.ts,
x=queued_value.channel_value.x,
y=queued_value.channel_value.y))
channels_with_values.append(ChannelIdWithValues(channel_id, channel_values))
try:
# pylint:disable=protected-access
self._experiment._send_channels_values(channels_with_values)
except HTTPUnprocessableEntity as e:
message = "Maximum storage limit reached"
try:
message = e.response.json()["message"]
finally:
_logger.warning('Failed to send channel value: %s', message)
except (NeptuneException, IOError):
_logger.exception('Failed to send channel value.')
def test_send_when_waiting_for_next_value_timed_out(self):
# given
numeric_values = [
ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3)
]
# and
semaphore = threading.Semaphore(0)
# pylint: disable=protected-access
self._EXPERIMENT._send_channels_values.side_effect = lambda _: semaphore.release(
)
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in numeric_values:
channels_values_sender.send(self._NUMERIC_CHANNEL.name,
self._NUMERIC_CHANNEL.channelType,
channel_value)
# then
# pylint: disable=protected-access
semaphore.acquire()
self._EXPERIMENT._send_channels_values.assert_called_with([
ChannelIdWithValues(channel_id=self._NUMERIC_CHANNEL.id,
channel_values=numeric_values)
])
# and
self._EXPERIMENT._send_channels_values.reset_mock()
channels_values_sender.join()
# and
self._EXPERIMENT._send_channels_values.assert_not_called()
def test_send_values_on_join(self):
# given
channel_value = ChannelValue(x=1, y="value", ts=self._TS)
# and
channels_values_sender = ChannelsValuesSender(experiment=self._EXPERIMENT)
# when
channels_values_sender.send(
self._TEXT_CHANNEL.name, self._TEXT_CHANNEL.channelType, channel_value
)
# and
channels_values_sender.join()
# then
# pylint: disable=protected-access
self._EXPERIMENT._send_channels_values.assert_called_with(
[
ChannelIdWithValues(
channel_id=self._TEXT_CHANNEL.id,
channel_name=self._TEXT_CHANNEL.name,
channel_type=self._TEXT_CHANNEL.channelType,
channel_namespace=ChannelNamespace.USER,
channel_values=[channel_value],
)
]
)
def test_send_when_waiting_for_next_value_timed_out(self):
# given
numeric_values = [
ChannelValue(x=i, y=i, ts=self._TS + i) for i in range(0, 3)
]
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in numeric_values:
channels_values_sender.send(self._NUMERIC_CHANNEL.name,
self._NUMERIC_CHANNEL.channelType,
channel_value)
# and
time.sleep(self.__TIMEOUT * 2)
# then
# pylint: disable=protected-access
self._EXPERIMENT._send_channels_values.assert_called_with([
ChannelIdWithValues(channel_id=self._NUMERIC_CHANNEL.id,
channel_values=numeric_values)
])
# and
self._EXPERIMENT._send_channels_values.reset_mock()
channels_values_sender.join()
# and
self._EXPERIMENT._send_channels_values.assert_not_called()
def _send_values(self, queued_channels_values):
channel_key = lambda value: (value.channel_name, value.channel_type)
queued_grouped_by_channel = {
channel: list(values)
for channel, values in groupby(
sorted(queued_channels_values, key=channel_key), channel_key)
}
channels_with_values = []
for (channel_name, channel_type) in queued_grouped_by_channel:
# pylint: disable=protected-access
channel = self._experiment._get_channel(channel_name, channel_type)
last_x = channel.x if channel.x else 0
channel_values = []
for queued_value in queued_grouped_by_channel[(channel_name,
channel_type)]:
x = queued_value.channel_value.x if queued_value.channel_value.x is not None else last_x + 1
channel_values.append(
ChannelValue(ts=queued_value.channel_value.ts,
x=x,
y=queued_value.channel_value.y))
last_x = x
channels_with_values.append(
ChannelIdWithValues(channel.id, channel_values))
# pylint: disable=protected-access
try:
self._experiment._send_channels_values(channels_with_values)
except (NeptuneApiException, IOError):
pass
def test_send_values_in_multiple_batches(self):
# given
channels_values = [
ChannelValue(x=i, y="value{}".format(i), ts=self._TS + i)
for i in range(0, self._BATCH_SIZE * 3)
]
# and
channels_values_sender = ChannelsValuesSender(
experiment=self._EXPERIMENT)
# when
for channel_value in channels_values:
channels_values_sender.send(self._TEXT_CHANNEL.name,
self._TEXT_CHANNEL.channelType,
channel_value)
# and
channels_values_sender.join()
# then
# pylint: disable=protected-access
self.assertEqual(self._EXPERIMENT._send_channels_values.mock_calls, [
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._TEXT_CHANNEL.id,
channel_values=channels_values[0:self._BATCH_SIZE])
]),
mock.call._send_channels_values([
ChannelIdWithValues(channel_id=self._TEXT_CHANNEL.id,
channel_values=channels_values[
self._BATCH_SIZE:self._BATCH_SIZE * 2])
]),
mock.call._send_channels_values([
ChannelIdWithValues(
channel_id=self._TEXT_CHANNEL.id,
channel_values=channels_values[self._BATCH_SIZE *
2:self._BATCH_SIZE * 3])
])
])
def _test_send_channel_values(
self,
channel_y_elements: List[tuple],
expected_operation: str,
channel_type: ChannelType,
):
# given prepared `ChannelIdWithValues`
channel_id = "channel_id"
channel_name = "channel_name"
now_ms = int(time.time() * 1000)
channel_with_values = ChannelIdWithValues(
channel_id=channel_id,
channel_name=channel_name,
channel_type=channel_type.value,
channel_namespace=ChannelNamespace.USER,
channel_values=[
ChannelValue(x=None,
y={channel_y_key: channel_y_value},
ts=None)
for channel_y_key, channel_y_value in channel_y_elements
],
)
# invoke send_channels_values
self.leaderboard.send_channels_values(self.exp_mock,
[channel_with_values])
# expect `executeOperations` was called once with properly prepared kwargs
expected_call_args = {
"experimentId":
"00000000-0000-0000-0000-000000000000",
"operations": [{
"path": f"logs/{channel_name}",
expected_operation: {
"entries": [{
"value": channel_y_value,
"step": None,
"timestampMilliseconds": now_ms,
} for _, channel_y_value in channel_y_elements]
},
}],
}
# pylint:disable=protected-access
execute_operations = (
self.leaderboard.leaderboard_swagger_client.api.executeOperations)
self.assertEqual(len(execute_operations.call_args_list), 1)
self.assertDictEqual(execute_operations.call_args_list[0][1],
expected_call_args)