def test_get_numeric_channels_values(self):
# when
client = MagicMock()
client.get_channel_points_csv.return_value = StringIO(u'\n'.join(['0.3,2.5', '1,2']))
experiment = MagicMock()
experiment.id = a_string()
experiment.internal_id = a_uuid_string()
experiment.channels = [Bunch(id=a_uuid_string(), name='epoch_loss')]
experiment.channelsLastValues = [Bunch(channelName='epoch_loss', x=2.5, y=2)]
client.get_experiment.return_value = experiment
# then
experiment = Experiment(
client=client,
_id=a_string(),
internal_id=a_uuid_string(),
project_full_id="test/sandbox"
)
result = experiment.get_numeric_channels_values('epoch_loss')
expected_result = pd.DataFrame({'x': [0.3, 1.0],
'epoch_loss': [2.5, 2.0]}, dtype=float)
expected_result = sort_df_by_columns(expected_result)
result = sort_df_by_columns(result)
assert_frame_equal(expected_result, result)
def test_get_numeric_channels_values(self):
# when
backend = MagicMock()
backend.get_channel_points_csv.return_value = StringIO(u"\n".join(
["0.3,2.5", "1,2"]))
experiment = MagicMock()
experiment.id = a_string()
experiment.internal_id = a_uuid_string()
experiment.channels = [Munch(id=a_uuid_string(), name="epoch_loss")]
experiment.channelsLastValues = [
Munch(channelName="epoch_loss", x=2.5, y=2)
]
backend.get_experiment.return_value = experiment
# then
experiment = Experiment(
backend=backend,
project=a_project(),
_id=a_string(),
internal_id=a_uuid_string(),
)
result = experiment.get_numeric_channels_values("epoch_loss")
expected_result = pd.DataFrame(
{
"x": [0.3, 1.0],
"epoch_loss": [2.5, 2.0]
}, dtype=float)
expected_result = sort_df_by_columns(expected_result)
result = sort_df_by_columns(result)
assert_frame_equal(expected_result, result)
def an_experiment_leaderboard_entry_dto():
entry = MagicMock()
entry.entryType = 'experiment'
entry.id = a_uuid_string()
entry.shortId = a_string()
entry.projectId = a_uuid_string()
entry.state = 'succeeded'
entry.experimentStates = an_experiment_states(succeeded=1)
entry.responding = True
entry.name = a_string()
entry.organizationName = a_string()
entry.projectName = a_string()
entry.description = a_string()
entry.timeOfCreation = a_timestamp()
entry.timeOfCompletion = a_timestamp()
entry.runningTime = randint(1, 1000)
entry.owner = a_string()
entry.size = randint(1, 1000)
entry.tags = [a_string(), a_string()]
entry.environment = a_string()
entry.workerType = a_string()
entry.hostname = a_string()
entry.sourceSize = randint(1, 1000)
entry.sourceMd5 = a_string()
entry.commitId = a_string()
entry.properties = [a_property(), a_property()]
entry.parameters = [a_parameter(), a_parameter()]
entry.channelsLastValues = [a_channel_value(), a_channel_value()]
entry.trashed = False
entry.deleted = False
entry.isBestExperiment = False
return entry
def test_append_tags(self):
# given
backend = mock.MagicMock()
experiment = Experiment(backend, a_project(), an_experiment_id(),
a_uuid_string())
# and
def build_call(tags_list):
return call(experiment=experiment,
tags_to_add=tags_list,
tags_to_delete=[])
# when
experiment.append_tag("tag")
experiment.append_tag(["tag1", "tag2", "tag3"])
experiment.append_tag("tag1", "tag2", "tag3")
experiment.append_tags("tag")
experiment.append_tags(["tag1", "tag2", "tag3"])
experiment.append_tags("tag1", "tag2", "tag3")
# then
backend.update_tags.assert_has_calls([
build_call(["tag"]),
build_call(["tag1", "tag2", "tag3"]),
build_call(["tag1", "tag2", "tag3"]),
build_call(["tag"]),
build_call(["tag1", "tag2", "tag3"]),
build_call(["tag1", "tag2", "tag3"]),
])
def test_pop_experiment_from_stack(self):
# given
first_experiment = Munch(internal_id=a_uuid_string())
second_experiment = Munch(internal_id=a_uuid_string())
# and
self.project._push_new_experiment(first_experiment)
# when
self.project._push_new_experiment(second_experiment)
# then
self.assertEqual(self.project._get_current_experiment(), second_experiment)
# and
self.assertEqual(self.project._pop_stopped_experiment(), second_experiment)
# and
self.assertEqual(self.project._get_current_experiment(), first_experiment)
def a_parameter():
p = MagicMock()
p.id = a_uuid_string()
p.name = a_string()
p.parameterType = 'double'
p.value = str(uniform(-100, 100))
return p
def test_delete_artifact(self):
# given
backend = mock.MagicMock()
project = a_project()
experiment = Experiment(
backend,
project,
an_experiment_id(),
a_uuid_string()
)
# and
def build_call(path):
return call(
experiment=experiment,
path=path
)
# when
experiment.delete_artifacts('/an_abs_path_in_exp_output')
experiment.delete_artifacts('/../an_abs_path_in_exp')
experiment.delete_artifacts('/../../an_abs_path_in_prj')
experiment.delete_artifacts('a_path_in_exp_output')
self.assertRaises(ValueError, experiment.delete_artifacts, 'test/../../a_path_outside_exp')
self.assertRaises(ValueError, experiment.delete_artifacts, '../a_path_outside_exp')
self.assertRaises(ValueError, experiment.delete_artifacts, "..")
# then
backend.rm_data.assert_has_calls([
build_call('/an_abs_path_in_exp_output'),
build_call('/../an_abs_path_in_exp'),
build_call('/../../an_abs_path_in_prj'),
build_call('a_path_in_exp_output'),
])
def test_send_image(self, ChannelsValuesSender, content):
# given
channels_values_sender = ChannelsValuesSender.return_value
experiment = Experiment(mock.MagicMock(), a_project(),
an_experiment_id(), a_uuid_string())
image_value = dict(
name=a_string(),
description=a_string(),
data=base64.b64encode(content()).decode("utf-8"),
)
channel_value = ChannelValue(x=random.randint(0, 100),
y=dict(image_value=image_value),
ts=time.time())
# when
experiment.send_image(
"errors",
channel_value.x,
"/tmp/img.png",
image_value["name"],
image_value["description"],
channel_value.ts,
)
# then
channels_values_sender.send.assert_called_with("errors",
ChannelType.IMAGE.value,
channel_value)
def setUp(self):
super(TestProject, self).setUp()
self.backend = MagicMock()
self.project = Project(backend=self.backend,
internal_id=a_uuid_string(),
namespace=a_string(),
name=a_string())
self.current_directory = os.getcwd()
def a_channel():
x = random.randint(0, 100)
return Munch(id=a_uuid_string(),
name=a_string(),
channelType='text',
lastX=x,
x=x,
y=a_string())
def a_channel_value():
cv = MagicMock()
cv.channelId = a_uuid_string()
cv.channelName = a_string()
cv.channelType = 'numeric'
cv.x = uniform(1, 100)
cv.y = str(uniform(1, 100))
return cv
def test_get_current_experiment_from_stack(self):
# given
experiment = Munch(internal_id=a_uuid_string())
# when
self.project._push_new_experiment(experiment)
# then
self.assertEqual(self.project._get_current_experiment(), experiment)
def an_invited_project_member():
invitation_info = MagicMock()
invitation_info.id = a_uuid_string()
invitation_info.email = a_string() + '@example.com'
project_member = MagicMock()
project_member.invitationInfo = invitation_info
project_member.registeredMemberInfo = None
project_member.role = 'member'
return project_member
def test_send_text(self, ChannelsValuesSender):
# given
channels_values_sender = ChannelsValuesSender.return_value
experiment = Experiment(mock.MagicMock(), a_project(),
an_experiment_id(), a_uuid_string())
channel_value = ChannelValue(x=random.randint(0, 100),
y=dict(text_value=a_string()),
ts=time.time())
# when
experiment.send_text("stdout", channel_value.x,
channel_value.y["text_value"], channel_value.ts)
# then
channels_values_sender.send.assert_called_with("stdout",
ChannelType.TEXT.value,
channel_value)
def test_send_metric(self, ChannelsValuesSender):
# given
channels_values_sender = ChannelsValuesSender.return_value
experiment = Experiment(mock.MagicMock(), mock.MagicMock(),
an_experiment_id(), a_uuid_string())
channel_value = ChannelValue(
x=random.randint(0, 100),
y=dict(numeric_value=random.randint(0, 100)),
ts=time.time())
# when
experiment.send_metric('loss', channel_value.x,
channel_value.y['numeric_value'],
channel_value.ts)
# then
channels_values_sender.send.assert_called_with(
'loss', ChannelType.NUMERIC.value, channel_value)
def setUp(self):
super(TestProject, self).setUp()
self.client = MagicMock()
self.project = Project(client=self.client, internal_id=a_uuid_string(), namespace=a_string(), name=a_string())
def a_project():
project = MagicMock()
project.id = a_uuid_string()
project.name = a_string()
return project
def setUp(self):
super(TestProject, self).setUp()
self.backend = MagicMock()
self.project = Project(backend=self.backend, internal_id=a_uuid_string(), namespace=a_string(), name=a_string())
