def test_stop_non_managed_run(self): experiment = RunFactory(project=self.project, user=self.user, is_managed=False) runs_stop(run_id=experiment.id, update_status=True) experiment.refresh_from_db() assert experiment.status == V1Statuses.STOPPED
def setUp(self): super().setUp() self.project = ProjectFactory() self.run = RunFactory( user=self.user, project=self.project, content="test", raw_content="test", is_managed=True, ) self.url = "/{}/{}/{}/runs/{}/artifacts_lineage/names/".format( API_V1, "polyaxon", self.project.name, self.run.uuid.hex) self.objects = [] for i in range(self.num_objects): obj = self.factory_class( name=f"foo{i}", state=uuid.uuid4(), ) self.objects.append(obj) ArtifactLineage.objects.create(run=self.run, artifact=obj) self.query = self.queryset.filter(run=self.run) # Other user objects other_project = ProjectFactory() other_run = RunFactory( user=self.user, project=other_project, content="test", raw_content="test", is_managed=True, ) other_obj = self.factory_class(state=uuid.uuid4(), ) ArtifactLineage.objects.create(run=other_run, artifact=other_obj)
def test_stop_managed_run(self, runs_prepare, managed_stop): managed_stop.return_value = True experiment = RunFactory(project=self.project, user=self.user, is_managed=True, raw_content="test") assert runs_prepare.call_count == 1 experiment.refresh_from_db() assert experiment.status == V1Statuses.CREATED runs_stop(run_id=experiment.id, update_status=True) assert managed_stop.call_count == 1
def test_stop_managed_wrong_stop_retries(self, mock_resolve, managed_stop, mock_stop_run): managed_stop.return_value = False experiment = RunFactory(project=self.project, user=self.user, is_managed=True, raw_content="test") runs_stop(run_id=experiment.id) assert mock_stop_run.call_count == 1 assert managed_stop.call_count == 1 experiment.refresh_from_db() assert experiment.status == V1Statuses.CREATED
def test_stop_managed_run(self, runs_prepare, managed_stop): with mock.patch("django.db.transaction.on_commit", lambda t: t()): managed_stop.return_value = True experiment = RunFactory(project=self.project, user=self.user, is_managed=True, raw_content="test") assert runs_prepare.call_count == 1 experiment.refresh_from_db() assert experiment.status == V1Statuses.CREATED runs_stop(run_id=experiment.id, update_status=True) assert managed_stop.call_count == 1
def test_prepare_run_of_already_failed_run_mock(self, mock_resolve, mock_prepare_run, mock_start_run): spec_run = MagicMock(cache=V1Cache(disable=True)) mock_resolve.return_value = (None, spec_run) experiment = RunFactory(project=self.project, user=self.user, raw_content="test", is_managed=True) # We are patching the automatic call and executing prepare manually runs_prepare(run_id=experiment.id) experiment.refresh_from_db() assert experiment.status == V1Statuses.COMPILED assert mock_prepare_run.call_count == 1 # Automatic call from executor assert mock_start_run.call_count == 1
def test_prepare_run_of_already_skipped_run(self, mock_resolve): spec_run = MagicMock(cache=V1Cache(disable=False)) mock_resolve.return_value = (None, spec_run) experiment = RunFactory(project=self.project, user=self.user) new_run_status( run=experiment, condition=V1StatusCondition.get_condition(type=V1Statuses.SKIPPED, status=True), ) new_experiment = RunFactory(project=self.project, user=self.user) runs_prepare(run_id=new_experiment.id) new_experiment.refresh_from_db() assert new_experiment.status == V1Statuses.COMPILED
def setUp(self): super().setUp() self.project = ProjectFactory() self.run = RunFactory( project=self.project, content="test", raw_content="test", is_managed=True, ) self.state = self.project.owner.uuid
def test_start_run(self, manager_start): experiment = RunFactory(project=self.project, user=self.user) new_run_status( run=experiment, condition=V1StatusCondition.get_condition(type=V1Statuses.COMPILED, status=True), ) runs_start(run_id=experiment.id) assert manager_start.call_count == 1
def test_delete(self): for _ in range(2): RunFactory(project=self.project, user=self.user) assert self.queryset.count() == 1 assert Run.objects.count() == 4 resp = self.client.delete(self.url) assert resp.status_code == status.HTTP_204_NO_CONTENT assert self.queryset.count() == 0 assert Project.all.filter().count() == 0 assert Run.all.count() == 0
def setUp(self): super().setUp() self.user = UserFactory() self.project = self.factory_class() self.url = "/{}/{}/{}/".format(API_V1, self.user.username, self.project.name) self.queryset = self.model_class.objects.filter() self.object_query = self.model_class.objects.get(id=self.project.id) # Create related fields for _ in range(2): RunFactory(user=self.user, project=self.project)
def test_create_run(self, auditor_record): run = RunFactory(project=self.project, user=self.user) assert auditor_record.call_count == 1 call_args, call_kwargs = auditor_record.call_args assert call_kwargs["event_type"] == run_events.RUN_CREATED assert run.user == self.user assert run.project == self.project assert run.name is None assert run.description is None assert run.content is None assert run.readme is None assert run.tags is None assert run.cloning_kind is None assert run.original is None
def setUp(self): super().setUp() self.project = ProjectFactory() self.run = RunFactory( user=self.user, project=self.project, content="test", raw_content="test", is_managed=True, ) self.artifact = self.factory_class(name="foo", state=self.project.owner.uuid) self.artifact_lineage = ArtifactLineage.objects.create( artifact=self.artifact, run=self.run) self.url = "/{}/{}/{}/runs/{}/artifacts_lineage/{}/".format( API_V1, "polyaxon", self.project.name, self.run.uuid.hex, self.artifact.name, ) self.queryset = Artifact.objects.all()
def setUp(self): super().setUp() self.project = ProjectFactory() self.objects = [ RunFactory(project=self.project, user=self.user) for _ in range(3) ] self.url = "/{}/polyaxon/{}/runs/artifacts_lineage/".format( API_V1, self.project.name, ) obj = self.factory_class(name="in1", state=self.project.uuid) ArtifactLineage.objects.create(run=self.objects[0], artifact=obj) ArtifactLineage.objects.create(run=self.objects[1], artifact=obj) obj = self.factory_class(name="in2", state=self.project.uuid) ArtifactLineage.objects.create(run=self.objects[0], artifact=obj) obj = self.factory_class(name="out1", state=self.project.uuid) ArtifactLineage.objects.create( run=self.objects[0], artifact=obj, is_input=False, ) self.query = self.queryset.filter(run__project=self.project)
def test_create_run_without_spec(self): run = RunFactory(project=self.project, user=self.user) assert run.name is None
def setUp(self): super().setUp() self.user = UserFactory() self.project = ProjectFactory() self.run = RunFactory(project=self.project)
def test_range_apply(self): # Delete current run self.run.delete() run = RunFactory( project=self.project, is_managed=False, outputs={ "accuracy": 0.9, "precision": 0.9 }, ) run.created_at = datetime.datetime(2018, 1, 1) run.save() run = RunFactory( project=self.project, is_managed=False, outputs={ "accuracy": 0.9, "precision": 0.9 }, ) run.created_at = datetime.datetime(2010, 1, 1) run.save() eq_cond = DateTimeCondition(op="eq") lt_cond = DateTimeCondition(op="lt") lte_cond = DateTimeCondition(op="lte") gt_cond = DateTimeCondition(op="gt") gte_cond = DateTimeCondition(op="gte") range_cond = DateTimeCondition(op="range") nrange_cond = DateTimeCondition(op="range", negation=True) # eq queryset = eq_cond.apply( queryset=Run.objects, name="created_at", params="2018-01-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = eq_cond.apply( queryset=Run.objects, name="created_at", params="2018-02-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 # lt queryset = lt_cond.apply( queryset=Run.objects, name="created_at", params="2018-02-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = lt_cond.apply( queryset=Run.objects, name="created_at", params="2018-01-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = lt_cond.apply( queryset=Run.objects, name="created_at", params="2008-01-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 # lte queryset = lte_cond.apply( queryset=Run.objects, name="created_at", params="2018-02-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = lte_cond.apply( queryset=Run.objects, name="created_at", params="2018-01-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = lte_cond.apply( queryset=Run.objects, name="created_at", params="2008-01-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 # gt queryset = gt_cond.apply( queryset=Run.objects, name="created_at", params="2018-02-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = gt_cond.apply( queryset=Run.objects, name="created_at", params="2018-01-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = gt_cond.apply( queryset=Run.objects, name="created_at", params="2008-01-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 # lte queryset = gte_cond.apply( queryset=Run.objects, name="created_at", params="2018-02-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = gte_cond.apply( queryset=Run.objects, name="created_at", params="2018-01-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = gte_cond.apply( queryset=Run.objects, name="created_at", params="2008-01-01", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 # range queryset = range_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2018-02-01 00:00", settings.TIME_ZONE), DateTimeFormatter.extract("2018-03-01 00:00", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = range_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2018-02-01", settings.TIME_ZONE), DateTimeFormatter.extract("2008-03-01", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = range_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2017-02-01", settings.TIME_ZONE), DateTimeFormatter.extract("2018-03-01", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = range_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2008-02-01 00:00:12", settings.TIME_ZONE), DateTimeFormatter.extract("2018-03-01", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = range_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2010-01-01", settings.TIME_ZONE), DateTimeFormatter.extract("2010-01-01", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 # nrange queryset = nrange_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2018-02-01 00:00", settings.TIME_ZONE), DateTimeFormatter.extract("2018-03-01 00:00", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = nrange_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2018-02-01", settings.TIME_ZONE), DateTimeFormatter.extract("2008-03-01", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = nrange_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2017-02-01", settings.TIME_ZONE), DateTimeFormatter.extract("2018-03-01", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = nrange_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2008-02-01 00:00:12", settings.TIME_ZONE), DateTimeFormatter.extract("2018-03-01", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = nrange_cond.apply( queryset=Run.objects, name="created_at", params=( DateTimeFormatter.extract("2010-01-01", settings.TIME_ZONE), DateTimeFormatter.extract("2010-01-01", settings.TIME_ZONE), ), query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2
def test_comparison_apply(self): self.run.inputs = {"rate": 1, "loss": "foo"} self.run.save() RunFactory(project=self.project, is_managed=False, outputs={ "loss": 0.1, "step": 1 }) RunFactory(project=self.project, is_managed=False, outputs={ "loss": 0.3, "step": 10 }) RunFactory(project=self.project, is_managed=False, inputs={ "rate": -1, "loss": "bar" }) RunFactory(project=self.project, is_managed=False, outputs={ "loss": 0.9, "step": 100 }) eq_cond = ComparisonCondition(op="eq") neq_cond = ComparisonCondition(op="eq", negation=True) lt_cond = ComparisonCondition(op="lt") lte_cond = ComparisonCondition(op="lte") gt_cond = ComparisonCondition(op="gt") gte_cond = ComparisonCondition(op="gte") # eq queryset = eq_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.1, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = eq_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.2, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = eq_cond.apply( queryset=Run.objects, name="outputs__step", params=10, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 # neq must use the table directly queryset = neq_cond.apply( queryset=Run.objects, name="inputs__rate", params=1, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 4 queryset = neq_cond.apply( queryset=Run.objects, name="inputs__rate", params=-1, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 4 queryset = neq_cond.apply( queryset=Run.objects, name="inputs__rate", params=-12, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 5 queryset = neq_cond.apply( queryset=Run.objects, name="inputs__loss", params="foo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 4 queryset = neq_cond.apply( queryset=Run.objects, name="inputs__loss", params="moo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 5 # lt queryset = lt_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.1, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = lt_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.2, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = lt_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.9, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 # lte queryset = lte_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.1, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = lte_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.2, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = lte_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.9, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 3 # gt queryset = gt_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.1, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = gt_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.2, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = gt_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.9, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 # gte queryset = gte_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.1, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 3 queryset = gte_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.2, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = gte_cond.apply( queryset=Run.objects, name="outputs__loss", params=0.9, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1
def test_create_run_without_content_passes(self): run = RunFactory(project=self.project) assert run.content is None assert run.is_managed is False
def test_create_run_with_content_and_is_managed(self): with self.assertRaises(ValidationError): RunFactory(project=self.project, is_managed=True, content="foo") RunFactory(project=self.project, is_managed=True, raw_content="foo")
class TestStatuses(TestCase): def setUp(self): super().setUp() self.user = UserFactory() self.project = ProjectFactory() self.run = RunFactory(project=self.project, user=self.user) def test_new_failed_status_after_stopping(self): new_run_status( self.run, condition=V1StatusCondition.get_condition(type=V1Statuses.STOPPING, status=True, reason="foo"), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 1 # Same this condition new_run_status( self.run, condition=V1StatusCondition.get_condition( type=V1Statuses.SCHEDULED, status=True, reason="foo"), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 1 # Different condition's message new_run_status( self.run, condition=V1StatusCondition.get_condition(type=V1Statuses.FAILED, status=True, reason="foo", message="New message"), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 2 def test_new_stopped_status_after_stopping(self): new_run_status( self.run, condition=V1StatusCondition.get_condition(type=V1Statuses.STOPPING, status=True, reason="foo"), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 1 # Same this condition new_run_status( self.run, condition=V1StatusCondition.get_condition(type=V1Statuses.RUNNING, status=True, reason="foo"), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 1 # Different condition's message new_run_status( self.run, condition=V1StatusCondition.get_condition( type=V1Statuses.STOPPED, status=True, reason="foo", message="New message", ), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 2 def test_new_status_equality(self): new_run_status( self.run, condition=V1StatusCondition.get_condition( type=V1Statuses.SCHEDULED, status=True, reason="foo"), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 1 # Same this condition new_run_status( self.run, condition=V1StatusCondition.get_condition( type=V1Statuses.SCHEDULED, status=True, reason="foo"), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 1 # Different condition's message new_run_status( self.run, condition=V1StatusCondition.get_condition( type=V1Statuses.SCHEDULED, status=True, reason="foo", message="New message", ), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 1 # New condition new_run_status( self.run, condition=V1StatusCondition.get_condition( type=V1Statuses.RUNNING, status=True, reason="foo", message="New message", ), ) self.run.refresh_from_db() assert len(self.run.status_conditions) == 2 def test_new_status_set_start_date(self): # No status change assert self.run.started_at is None new_run_status( self.run, condition=V1StatusCondition.get_condition(type=V1Statuses.CREATED, status=True), ) assert self.run.started_at is None assert self.run.finished_at is None assert self.run.run_time is None new_run_status( self.run, condition=V1StatusCondition.get_condition( type=V1Statuses.SCHEDULED, status=True), ) self.run.refresh_from_db() assert self.run.started_at is None assert self.run.finished_at is None assert self.run.run_time is None # Set a running status self.run.status = V1Statuses.CREATED new_run_status( self.run, condition=V1StatusCondition.get_condition(type=V1Statuses.STARTING, status=True), ) self.run.refresh_from_db() assert self.run.started_at is not None assert self.run.finished_at is None assert self.run.run_time is None started_at = self.run.started_at self.run.status = V1Statuses.CREATED new_run_status( self.run, condition=V1StatusCondition.get_condition(type=V1Statuses.RUNNING, status=True), ) assert self.run.started_at == started_at assert self.run.finished_at is None assert self.run.run_time is None assert len(self.run.status_conditions) == 3 condition1 = V1StatusCondition.get_condition( type=V1Statuses.STOPPED, status="True", reason="Run is stopped", message="zombie error", ) new_run_status(self.run, condition1) self.run.refresh_from_db() assert self.run.started_at == started_at assert self.run.finished_at is not None finished_at = self.run.finished_at assert self.run.run_time == (self.run.finished_at - self.run.started_at).seconds assert len(self.run.status_conditions) == 4 assert self.run.status_conditions[3]["type"] == V1Statuses.STOPPED assert self.run.status_conditions[3]["message"] == "zombie error" assert self.run.status_conditions[3]["reason"] == "Run is stopped" condition2 = V1StatusCondition.get_condition( type=V1Statuses.FAILED, status="True", reason="Run failed", message="some error", ) new_run_status(self.run, condition2) self.run.refresh_from_db() assert self.run.started_at == started_at assert self.run.finished_at == finished_at assert self.run.run_time == (self.run.finished_at - self.run.started_at).seconds assert len(self.run.status_conditions) == 5 assert self.run.status_conditions[3]["type"] == V1Statuses.STOPPED assert self.run.status_conditions[4]["type"] == V1Statuses.FAILED # Update the stopped status condition3 = V1StatusCondition.get_condition( type=V1Statuses.STOPPED, status="True", reason="Run failed", message="some error", ) new_run_status(self.run, condition3) self.run.refresh_from_db() assert self.run.started_at == started_at assert self.run.finished_at == finished_at assert self.run.run_time == (self.run.finished_at - self.run.started_at).seconds assert len(self.run.status_conditions) == 6 assert self.run.status_conditions[3]["type"] == V1Statuses.STOPPED assert self.run.status_conditions[4]["type"] == V1Statuses.FAILED assert self.run.status_conditions[5]["type"] == V1Statuses.STOPPED assert self.run.status_conditions[5]["message"] == "some error" assert self.run.status_conditions[5]["reason"] == "Run failed"
def test_bulk_run_status(self): run1 = RunFactory(project=self.project, kind=V1RunKind.JOB) run2 = RunFactory(project=self.project, kind=V1RunKind.JOB) run3 = RunFactory(project=self.project, kind=V1RunKind.SERVICE) # Patch all runs to be managed Run.all.update(is_managed=True) assert run1.status != V1Statuses.QUEUED assert run2.status != V1Statuses.QUEUED assert run3.status != V1Statuses.QUEUED condition = V1StatusCondition.get_condition( type=V1Statuses.QUEUED, status="True", reason="PolyaxonRunQueued", message="Run is queued", ) bulk_new_run_status([run1, run2, run3], condition) run1.refresh_from_db() assert run1.status == V1Statuses.QUEUED run2.refresh_from_db() assert run2.status == V1Statuses.QUEUED run3.refresh_from_db() assert run3.status == V1Statuses.QUEUED
def test_create_run_with_no_spec_and_params(self): run = RunFactory(project=self.project, content=None) assert run.content is None
def test_runs_set_artifacts(self): experiment = RunFactory(project=self.project, user=self.user) state = experiment.uuid assert experiment.artifacts.count() == 0 metric1 = V1RunArtifact( name="accuracy", kind=V1ArtifactKind.METRIC, path="accuracy", summary=dict(last_value=0.77, max_value=0.99, min_value=0.1, max_step=100), ) metric2 = V1RunArtifact( name="precision", kind=V1ArtifactKind.METRIC, path="precision", state=state, summary=dict(last_value=0.8, max_value=0.99, min_value=0.11, max_step=100), ) runs_set_artifacts(run_id=experiment.id, artifacts=[metric1.to_dict(), metric2.to_dict()]) assert experiment.artifacts.count() == 2 results = { r.name: V1RunArtifact.from_model(r) for r in Artifact.objects.all() } result1 = results["accuracy"].to_dict() # State is generated assert result1.pop("state") is not None assert result1 == metric1.to_dict() result2 = results["precision"].to_dict() # State is the same assert result2 == metric2.to_dict() metric1 = V1RunArtifact( name="accuracy", kind=V1ArtifactKind.METRIC, path="accuracy", state=state, summary=dict(last_value=0.8, max_value=0.99, min_value=0.1, max_step=100), ) metric3 = V1RunArtifact( name="recall", kind=V1ArtifactKind.METRIC, path="recall", state=state, summary=dict(last_value=0.1, max_value=0.2, min_value=0.1, max_step=100), ) runs_set_artifacts(run_id=experiment.id, artifacts=[metric1.to_dict(), metric3.to_dict()]) assert experiment.artifacts.count() == 3 results = { r.name: V1RunArtifact.from_model(r) for r in Artifact.objects.all() } assert results["accuracy"].to_dict() == metric1.to_dict() assert results["precision"].to_dict() == metric2.to_dict() assert results["recall"].to_dict() == metric3.to_dict()
def test_create_run_without_content_and_managed_raises(self): with self.assertRaises(ValidationError): RunFactory(project=self.project, is_managed=True)
def test_range_apply(self): new_run_status( self.run, condition=V1StatusCondition.get_condition(type=V1Statuses.FAILED, status=True), ) run2 = RunFactory(project=self.project) new_run_status( run2, condition=V1StatusCondition.get_condition(type=V1Statuses.STOPPED, status=True), ) run3 = RunFactory(project=self.project) new_run_status( run3, condition=V1StatusCondition.get_condition(type=V1Statuses.RUNNING, status=True), ) eq_cond = ValueCondition(op="eq") neq_cond = ValueCondition(op="eq", negation=True) in_cond = ValueCondition(op="in") nin_cond = ValueCondition(op="in", negation=True) # eq queryset = eq_cond.apply( queryset=Run.objects, name="status", params=V1Statuses.STOPPED, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = eq_cond.apply( queryset=Run.objects, name="status", params="foo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 # neq queryset = neq_cond.apply( queryset=Run.objects, name="status", params=V1Statuses.STOPPED, query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = neq_cond.apply( queryset=Run.objects, name="status", params="doo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 3 # in queryset = in_cond.apply( queryset=Run.objects, name="status", params=[V1Statuses.STOPPED, V1Statuses.RUNNING], query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = in_cond.apply( queryset=Run.objects, name="status", params=[V1Statuses.STOPPED, V1Statuses.RESUMING], query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = in_cond.apply( queryset=Run.objects, name="status", params=[V1Statuses.RESUMING, V1Statuses.SKIPPED], query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = in_cond.apply( queryset=Run.objects, name="status", params=[V1Statuses.FAILED, V1Statuses.STOPPED, V1Statuses.RUNNING], query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 3 # nin queryset = nin_cond.apply( queryset=Run.objects, name="status", params=[V1Statuses.STOPPED, V1Statuses.RUNNING], query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = nin_cond.apply( queryset=Run.objects, name="status", params=[V1Statuses.STOPPED, V1Statuses.RESUMING], query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = nin_cond.apply( queryset=Run.objects, name="status", params=[V1Statuses.RESUMING, V1Statuses.SKIPPED], query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 3 queryset = nin_cond.apply( queryset=Run.objects, name="status", params=[V1Statuses.FAILED, V1Statuses.STOPPED, V1Statuses.RUNNING], query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0
def test_creation_with_bad_config(self): run = RunFactory(project=self.project, content="foo") assert run.status == V1Statuses.CREATED assert run.content == "foo"
def test_range_apply(self): RunFactory(project=self.project, name="foo_bar") RunFactory(project=self.project, name="foo_moo") self.run.name = "moo_boo" self.run.save() contains_cond = SearchCondition(op="icontains") ncontains_cond = SearchCondition(op="icontains", negation=True) startswith_cond = SearchCondition(op="istartswith") nstartswith_cond = SearchCondition(op="istartswith", negation=True) endswith_cond = SearchCondition(op="iendswith") nendswith_cond = SearchCondition(op="iendswith", negation=True) # contains queryset = contains_cond.apply( queryset=Run.objects, name="name", params="foo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = contains_cond.apply( queryset=Run.objects, name="name", params="bar", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = contains_cond.apply( queryset=Run.objects, name="name", params="boo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = contains_cond.apply( queryset=Run.objects, name="name", params="none", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 # ncontains queryset = ncontains_cond.apply( queryset=Run.objects, name="name", params="foo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = ncontains_cond.apply( queryset=Run.objects, name="name", params="bar", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = ncontains_cond.apply( queryset=Run.objects, name="name", params="boo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = ncontains_cond.apply( queryset=Run.objects, name="name", params="none", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 3 # startswith queryset = startswith_cond.apply( queryset=Run.objects, name="name", params="foo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = startswith_cond.apply( queryset=Run.objects, name="name", params="bar", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = startswith_cond.apply( queryset=Run.objects, name="name", params="moo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 # nstartswith queryset = nstartswith_cond.apply( queryset=Run.objects, name="name", params="foo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = nstartswith_cond.apply( queryset=Run.objects, name="name", params="bar", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 3 queryset = nstartswith_cond.apply( queryset=Run.objects, name="name", params="moo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 # endswith queryset = endswith_cond.apply( queryset=Run.objects, name="name", params="foo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 0 queryset = endswith_cond.apply( queryset=Run.objects, name="name", params="bar", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 queryset = endswith_cond.apply( queryset=Run.objects, name="name", params="moo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 1 # nendswith queryset = nendswith_cond.apply( queryset=Run.objects, name="name", params="foo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 3 queryset = nendswith_cond.apply( queryset=Run.objects, name="name", params="bar", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2 queryset = nendswith_cond.apply( queryset=Run.objects, name="name", params="moo", query_backend=Q, timezone=settings.TIME_ZONE, ) assert queryset.count() == 2