def setUp(self):
super().setUp()
self.project = ProjectFactory()
self.run = RunFactory(
user=self.user,
project=self.project,
content="test",
raw_content="test",
is_managed=True,
)
self.url = "/{}/{}/{}/runs/{}/artifacts_lineage/names/".format(
API_V1, "polyaxon", self.project.name, self.run.uuid.hex)
self.objects = []
for i in range(self.num_objects):
obj = self.factory_class(
name=f"foo{i}",
state=uuid.uuid4(),
)
self.objects.append(obj)
ArtifactLineage.objects.create(run=self.run, artifact=obj)
self.query = self.queryset.filter(run=self.run)
# Other user objects
other_project = ProjectFactory()
other_run = RunFactory(
user=self.user,
project=other_project,
content="test",
raw_content="test",
is_managed=True,
)
other_obj = self.factory_class(state=uuid.uuid4(), )
ArtifactLineage.objects.create(run=other_run, artifact=other_obj)
def test_stop_non_managed_run(self):
experiment = RunFactory(project=self.project,
user=self.user,
is_managed=False)
runs_stop(run_id=experiment.id, update_status=True)
experiment.refresh_from_db()
assert experiment.status == V1Statuses.STOPPED
def test_prepare_run_of_already_skipped_run(self, mock_resolve):
spec_run = MagicMock(cache=V1Cache(disable=False))
mock_resolve.return_value = (None, spec_run)
experiment = RunFactory(project=self.project, user=self.user)
new_run_status(
run=experiment,
condition=V1StatusCondition.get_condition(type=V1Statuses.SKIPPED,
status=True),
)
new_experiment = RunFactory(project=self.project, user=self.user)
runs_prepare(run_id=new_experiment.id)
new_experiment.refresh_from_db()
assert new_experiment.status == V1Statuses.COMPILED
def setUp(self):
super().setUp()
self.project = ProjectFactory()
self.run = RunFactory(
project=self.project, content="test", raw_content="test", is_managed=True,
)
self.state = self.project.owner.uuid
def test_start_run(self, manager_start):
experiment = RunFactory(project=self.project, user=self.user)
new_run_status(
run=experiment,
condition=V1StatusCondition.get_condition(type=V1Statuses.COMPILED,
status=True),
)
runs_start(run_id=experiment.id)
assert manager_start.call_count == 1
def test_stop_managed_run(self, runs_prepare, managed_stop):
managed_stop.return_value = True
experiment = RunFactory(project=self.project,
user=self.user,
is_managed=True,
raw_content="test")
assert runs_prepare.call_count == 1
experiment.refresh_from_db()
assert experiment.status == V1Statuses.CREATED
runs_stop(run_id=experiment.id, update_status=True)
assert managed_stop.call_count == 1
def test_stop_managed_wrong_stop_retries(self, mock_resolve, managed_stop,
mock_stop_run):
managed_stop.return_value = False
experiment = RunFactory(project=self.project,
user=self.user,
is_managed=True,
raw_content="test")
runs_stop(run_id=experiment.id)
assert mock_stop_run.call_count == 1
assert managed_stop.call_count == 1
experiment.refresh_from_db()
assert experiment.status == V1Statuses.CREATED
def test_stop_managed_run(self, runs_prepare, managed_stop):
with mock.patch("django.db.transaction.on_commit", lambda t: t()):
managed_stop.return_value = True
experiment = RunFactory(project=self.project,
user=self.user,
is_managed=True,
raw_content="test")
assert runs_prepare.call_count == 1
experiment.refresh_from_db()
assert experiment.status == V1Statuses.CREATED
runs_stop(run_id=experiment.id, update_status=True)
assert managed_stop.call_count == 1
def setUp(self):
super().setUp()
self.user = UserFactory()
self.project = self.factory_class()
self.url = "/{}/{}/{}/".format(API_V1, self.user.username,
self.project.name)
self.queryset = self.model_class.objects.filter()
self.object_query = self.model_class.objects.get(id=self.project.id)
# Create related fields
for _ in range(2):
RunFactory(user=self.user, project=self.project)
def test_delete(self):
for _ in range(2):
RunFactory(project=self.project, user=self.user)
assert self.queryset.count() == 1
assert Run.objects.count() == 4
resp = self.client.delete(self.url)
assert resp.status_code == status.HTTP_204_NO_CONTENT
assert self.queryset.count() == 0
assert Project.all.filter().count() == 0
assert Run.all.count() == 0
def test_bulk_run_status(self):
run1 = RunFactory(project=self.project, kind=V1RunKind.JOB)
run2 = RunFactory(project=self.project, kind=V1RunKind.JOB)
run3 = RunFactory(project=self.project, kind=V1RunKind.SERVICE)
# Patch all runs to be managed
Run.all.update(is_managed=True)
assert run1.status != V1Statuses.QUEUED
assert run2.status != V1Statuses.QUEUED
assert run3.status != V1Statuses.QUEUED
condition = V1StatusCondition.get_condition(
type=V1Statuses.QUEUED,
status="True",
reason="PolyaxonRunQueued",
message="Run is queued",
)
bulk_new_run_status([run1, run2, run3], condition)
run1.refresh_from_db()
assert run1.status == V1Statuses.QUEUED
run2.refresh_from_db()
assert run2.status == V1Statuses.QUEUED
run3.refresh_from_db()
assert run3.status == V1Statuses.QUEUED
def test_create_run(self, auditor_record):
run = RunFactory(project=self.project, user=self.user)
assert auditor_record.call_count == 1
call_args, call_kwargs = auditor_record.call_args
assert call_kwargs["event_type"] == run_events.RUN_CREATED
assert run.user == self.user
assert run.project == self.project
assert run.name is None
assert run.description is None
assert run.content is None
assert run.readme is None
assert run.tags is None
assert run.cloning_kind is None
assert run.original is None
def test_prepare_run_of_already_failed_run_mock(self, mock_resolve,
mock_prepare_run,
mock_start_run):
spec_run = MagicMock(cache=V1Cache(disable=True))
mock_resolve.return_value = (None, spec_run)
experiment = RunFactory(project=self.project,
user=self.user,
raw_content="test",
is_managed=True)
# We are patching the automatic call and executing prepare manually
runs_prepare(run_id=experiment.id)
experiment.refresh_from_db()
assert experiment.status == V1Statuses.COMPILED
assert mock_prepare_run.call_count == 1 # Automatic call from executor
assert mock_start_run.call_count == 1
def setUp(self):
super().setUp()
self.project = ProjectFactory()
self.run = RunFactory(
user=self.user,
project=self.project,
content="test",
raw_content="test",
is_managed=True,
)
self.artifact = self.factory_class(name="foo",
state=self.project.owner.uuid)
self.artifact_lineage = ArtifactLineage.objects.create(
artifact=self.artifact, run=self.run)
self.url = "/{}/{}/{}/runs/{}/artifacts_lineage/{}/".format(
API_V1,
"polyaxon",
self.project.name,
self.run.uuid.hex,
self.artifact.name,
)
self.queryset = Artifact.objects.all()
def setUp(self):
super().setUp()
self.project = ProjectFactory()
self.objects = [
RunFactory(project=self.project, user=self.user) for _ in range(3)
]
self.url = "/{}/polyaxon/{}/runs/artifacts_lineage/".format(
API_V1,
self.project.name,
)
obj = self.factory_class(name="in1", state=self.project.uuid)
ArtifactLineage.objects.create(run=self.objects[0], artifact=obj)
ArtifactLineage.objects.create(run=self.objects[1], artifact=obj)
obj = self.factory_class(name="in2", state=self.project.uuid)
ArtifactLineage.objects.create(run=self.objects[0], artifact=obj)
obj = self.factory_class(name="out1", state=self.project.uuid)
ArtifactLineage.objects.create(
run=self.objects[0],
artifact=obj,
is_input=False,
)
self.query = self.queryset.filter(run__project=self.project)
def test_create_run_with_no_spec_and_params(self):
run = RunFactory(project=self.project, content=None)
assert run.content is None
def setUp(self):
super().setUp()
self.user = UserFactory()
self.project = ProjectFactory()
self.run = RunFactory(project=self.project)
def test_create_run_without_spec(self):
run = RunFactory(project=self.project, user=self.user)
assert run.name is None
def test_range_apply(self):
# Delete current run
self.run.delete()
run = RunFactory(
project=self.project,
is_managed=False,
outputs={
"accuracy": 0.9,
"precision": 0.9
},
)
run.created_at = datetime.datetime(2018, 1, 1)
run.save()
run = RunFactory(
project=self.project,
is_managed=False,
outputs={
"accuracy": 0.9,
"precision": 0.9
},
)
run.created_at = datetime.datetime(2010, 1, 1)
run.save()
eq_cond = DateTimeCondition(op="eq")
lt_cond = DateTimeCondition(op="lt")
lte_cond = DateTimeCondition(op="lte")
gt_cond = DateTimeCondition(op="gt")
gte_cond = DateTimeCondition(op="gte")
range_cond = DateTimeCondition(op="range")
nrange_cond = DateTimeCondition(op="range", negation=True)
# eq
queryset = eq_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-01-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = eq_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-02-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
# lt
queryset = lt_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-02-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = lt_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-01-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = lt_cond.apply(
queryset=Run.objects,
name="created_at",
params="2008-01-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
# lte
queryset = lte_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-02-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = lte_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-01-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = lte_cond.apply(
queryset=Run.objects,
name="created_at",
params="2008-01-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
# gt
queryset = gt_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-02-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = gt_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-01-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = gt_cond.apply(
queryset=Run.objects,
name="created_at",
params="2008-01-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
# lte
queryset = gte_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-02-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = gte_cond.apply(
queryset=Run.objects,
name="created_at",
params="2018-01-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = gte_cond.apply(
queryset=Run.objects,
name="created_at",
params="2008-01-01",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
# range
queryset = range_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2018-02-01 00:00",
settings.TIME_ZONE),
DateTimeFormatter.extract("2018-03-01 00:00",
settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = range_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2018-02-01", settings.TIME_ZONE),
DateTimeFormatter.extract("2008-03-01", settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = range_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2017-02-01", settings.TIME_ZONE),
DateTimeFormatter.extract("2018-03-01", settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = range_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2008-02-01 00:00:12",
settings.TIME_ZONE),
DateTimeFormatter.extract("2018-03-01", settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = range_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2010-01-01", settings.TIME_ZONE),
DateTimeFormatter.extract("2010-01-01", settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
# nrange
queryset = nrange_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2018-02-01 00:00",
settings.TIME_ZONE),
DateTimeFormatter.extract("2018-03-01 00:00",
settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = nrange_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2018-02-01", settings.TIME_ZONE),
DateTimeFormatter.extract("2008-03-01", settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = nrange_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2017-02-01", settings.TIME_ZONE),
DateTimeFormatter.extract("2018-03-01", settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = nrange_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2008-02-01 00:00:12",
settings.TIME_ZONE),
DateTimeFormatter.extract("2018-03-01", settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = nrange_cond.apply(
queryset=Run.objects,
name="created_at",
params=(
DateTimeFormatter.extract("2010-01-01", settings.TIME_ZONE),
DateTimeFormatter.extract("2010-01-01", settings.TIME_ZONE),
),
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
def test_comparison_apply(self):
self.run.inputs = {"rate": 1, "loss": "foo"}
self.run.save()
RunFactory(project=self.project,
is_managed=False,
outputs={
"loss": 0.1,
"step": 1
})
RunFactory(project=self.project,
is_managed=False,
outputs={
"loss": 0.3,
"step": 10
})
RunFactory(project=self.project,
is_managed=False,
inputs={
"rate": -1,
"loss": "bar"
})
RunFactory(project=self.project,
is_managed=False,
outputs={
"loss": 0.9,
"step": 100
})
eq_cond = ComparisonCondition(op="eq")
neq_cond = ComparisonCondition(op="eq", negation=True)
lt_cond = ComparisonCondition(op="lt")
lte_cond = ComparisonCondition(op="lte")
gt_cond = ComparisonCondition(op="gt")
gte_cond = ComparisonCondition(op="gte")
# eq
queryset = eq_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.1,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = eq_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.2,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = eq_cond.apply(
queryset=Run.objects,
name="outputs__step",
params=10,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
# neq must use the table directly
queryset = neq_cond.apply(
queryset=Run.objects,
name="inputs__rate",
params=1,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 4
queryset = neq_cond.apply(
queryset=Run.objects,
name="inputs__rate",
params=-1,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 4
queryset = neq_cond.apply(
queryset=Run.objects,
name="inputs__rate",
params=-12,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 5
queryset = neq_cond.apply(
queryset=Run.objects,
name="inputs__loss",
params="foo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 4
queryset = neq_cond.apply(
queryset=Run.objects,
name="inputs__loss",
params="moo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 5
# lt
queryset = lt_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.1,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = lt_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.2,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = lt_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.9,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
# lte
queryset = lte_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.1,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = lte_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.2,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = lte_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.9,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 3
# gt
queryset = gt_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.1,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = gt_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.2,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = gt_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.9,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
# gte
queryset = gte_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.1,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 3
queryset = gte_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.2,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = gte_cond.apply(
queryset=Run.objects,
name="outputs__loss",
params=0.9,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
def test_range_apply(self):
RunFactory(project=self.project, name="foo_bar")
RunFactory(project=self.project, name="foo_moo")
self.run.name = "moo_boo"
self.run.save()
contains_cond = SearchCondition(op="icontains")
ncontains_cond = SearchCondition(op="icontains", negation=True)
startswith_cond = SearchCondition(op="istartswith")
nstartswith_cond = SearchCondition(op="istartswith", negation=True)
endswith_cond = SearchCondition(op="iendswith")
nendswith_cond = SearchCondition(op="iendswith", negation=True)
# contains
queryset = contains_cond.apply(
queryset=Run.objects,
name="name",
params="foo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = contains_cond.apply(
queryset=Run.objects,
name="name",
params="bar",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = contains_cond.apply(
queryset=Run.objects,
name="name",
params="boo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = contains_cond.apply(
queryset=Run.objects,
name="name",
params="none",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
# ncontains
queryset = ncontains_cond.apply(
queryset=Run.objects,
name="name",
params="foo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = ncontains_cond.apply(
queryset=Run.objects,
name="name",
params="bar",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = ncontains_cond.apply(
queryset=Run.objects,
name="name",
params="boo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = ncontains_cond.apply(
queryset=Run.objects,
name="name",
params="none",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 3
# startswith
queryset = startswith_cond.apply(
queryset=Run.objects,
name="name",
params="foo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = startswith_cond.apply(
queryset=Run.objects,
name="name",
params="bar",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = startswith_cond.apply(
queryset=Run.objects,
name="name",
params="moo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
# nstartswith
queryset = nstartswith_cond.apply(
queryset=Run.objects,
name="name",
params="foo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = nstartswith_cond.apply(
queryset=Run.objects,
name="name",
params="bar",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 3
queryset = nstartswith_cond.apply(
queryset=Run.objects,
name="name",
params="moo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
# endswith
queryset = endswith_cond.apply(
queryset=Run.objects,
name="name",
params="foo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = endswith_cond.apply(
queryset=Run.objects,
name="name",
params="bar",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = endswith_cond.apply(
queryset=Run.objects,
name="name",
params="moo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
# nendswith
queryset = nendswith_cond.apply(
queryset=Run.objects,
name="name",
params="foo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 3
queryset = nendswith_cond.apply(
queryset=Run.objects,
name="name",
params="bar",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = nendswith_cond.apply(
queryset=Run.objects,
name="name",
params="moo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
def test_create_run_with_content_and_is_managed(self):
with self.assertRaises(ValidationError):
RunFactory(project=self.project, is_managed=True, content="foo")
RunFactory(project=self.project, is_managed=True, raw_content="foo")
def test_create_run_without_content_passes(self):
run = RunFactory(project=self.project)
assert run.content is None
assert run.is_managed is False
def test_create_run_without_content_and_managed_raises(self):
with self.assertRaises(ValidationError):
RunFactory(project=self.project, is_managed=True)
def test_runs_set_artifacts(self):
experiment = RunFactory(project=self.project, user=self.user)
state = experiment.uuid
assert experiment.artifacts.count() == 0
metric1 = V1RunArtifact(
name="accuracy",
kind=V1ArtifactKind.METRIC,
path="accuracy",
summary=dict(last_value=0.77,
max_value=0.99,
min_value=0.1,
max_step=100),
)
metric2 = V1RunArtifact(
name="precision",
kind=V1ArtifactKind.METRIC,
path="precision",
state=state,
summary=dict(last_value=0.8,
max_value=0.99,
min_value=0.11,
max_step=100),
)
runs_set_artifacts(run_id=experiment.id,
artifacts=[metric1.to_dict(),
metric2.to_dict()])
assert experiment.artifacts.count() == 2
results = {
r.name: V1RunArtifact.from_model(r)
for r in Artifact.objects.all()
}
result1 = results["accuracy"].to_dict()
# State is generated
assert result1.pop("state") is not None
assert result1 == metric1.to_dict()
result2 = results["precision"].to_dict()
# State is the same
assert result2 == metric2.to_dict()
metric1 = V1RunArtifact(
name="accuracy",
kind=V1ArtifactKind.METRIC,
path="accuracy",
state=state,
summary=dict(last_value=0.8,
max_value=0.99,
min_value=0.1,
max_step=100),
)
metric3 = V1RunArtifact(
name="recall",
kind=V1ArtifactKind.METRIC,
path="recall",
state=state,
summary=dict(last_value=0.1,
max_value=0.2,
min_value=0.1,
max_step=100),
)
runs_set_artifacts(run_id=experiment.id,
artifacts=[metric1.to_dict(),
metric3.to_dict()])
assert experiment.artifacts.count() == 3
results = {
r.name: V1RunArtifact.from_model(r)
for r in Artifact.objects.all()
}
assert results["accuracy"].to_dict() == metric1.to_dict()
assert results["precision"].to_dict() == metric2.to_dict()
assert results["recall"].to_dict() == metric3.to_dict()
def test_creation_with_bad_config(self):
run = RunFactory(project=self.project, content="foo")
assert run.status == V1Statuses.CREATED
assert run.content == "foo"
def test_range_apply(self):
new_run_status(
self.run,
condition=V1StatusCondition.get_condition(type=V1Statuses.FAILED,
status=True),
)
run2 = RunFactory(project=self.project)
new_run_status(
run2,
condition=V1StatusCondition.get_condition(type=V1Statuses.STOPPED,
status=True),
)
run3 = RunFactory(project=self.project)
new_run_status(
run3,
condition=V1StatusCondition.get_condition(type=V1Statuses.RUNNING,
status=True),
)
eq_cond = ValueCondition(op="eq")
neq_cond = ValueCondition(op="eq", negation=True)
in_cond = ValueCondition(op="in")
nin_cond = ValueCondition(op="in", negation=True)
# eq
queryset = eq_cond.apply(
queryset=Run.objects,
name="status",
params=V1Statuses.STOPPED,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = eq_cond.apply(
queryset=Run.objects,
name="status",
params="foo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
# neq
queryset = neq_cond.apply(
queryset=Run.objects,
name="status",
params=V1Statuses.STOPPED,
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = neq_cond.apply(
queryset=Run.objects,
name="status",
params="doo",
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 3
# in
queryset = in_cond.apply(
queryset=Run.objects,
name="status",
params=[V1Statuses.STOPPED, V1Statuses.RUNNING],
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = in_cond.apply(
queryset=Run.objects,
name="status",
params=[V1Statuses.STOPPED, V1Statuses.RESUMING],
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = in_cond.apply(
queryset=Run.objects,
name="status",
params=[V1Statuses.RESUMING, V1Statuses.SKIPPED],
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0
queryset = in_cond.apply(
queryset=Run.objects,
name="status",
params=[V1Statuses.FAILED, V1Statuses.STOPPED, V1Statuses.RUNNING],
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 3
# nin
queryset = nin_cond.apply(
queryset=Run.objects,
name="status",
params=[V1Statuses.STOPPED, V1Statuses.RUNNING],
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 1
queryset = nin_cond.apply(
queryset=Run.objects,
name="status",
params=[V1Statuses.STOPPED, V1Statuses.RESUMING],
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 2
queryset = nin_cond.apply(
queryset=Run.objects,
name="status",
params=[V1Statuses.RESUMING, V1Statuses.SKIPPED],
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 3
queryset = nin_cond.apply(
queryset=Run.objects,
name="status",
params=[V1Statuses.FAILED, V1Statuses.STOPPED, V1Statuses.RUNNING],
query_backend=Q,
timezone=settings.TIME_ZONE,
)
assert queryset.count() == 0