def test_create_from_task_fail_user_inputs(self): # Setup task # Create environment definition env_def_path = os.path.join(self.temp_dir, "Dockerfile") with open(env_def_path, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) task_obj = run("sh -c echo accuracy:0.45") # Test if failure if user gives environment_id with task_id failed = False try: _ = create(message="my test snapshot", task_id=task_obj.id, label="best", config={"foo": "bar"}, stats={"foo": "bar"}, environment_id="test_id") except SnapshotCreateFromTaskArgs: failed = True assert failed # Test if failure if user gives filepaths with task_id failed = False try: _ = create(message="my test snapshot", task_id=task_obj.id, label="best", config={"foo": "bar"}, stats={"foo": "bar"}, paths=["mypath"]) except SnapshotCreateFromTaskArgs: failed = True assert failed
def test_ls(self): # check project is not initialized if wrong home Config().set_home(os.path.join("does", "not", "exist")) failed = False try: ls() except InvalidProjectPath: failed = True assert failed # check session does not exist if wrong session Config().set_home(self.temp_dir) failed = False try: ls(session_id="does_not_exist") except SessionDoesNotExist: failed = True assert failed # create with default params and files to commit test_filepath = os.path.join(self.temp_dir, "script.py") with open(test_filepath, "wb") as f: f.write(to_bytes("import numpy\n")) f.write(to_bytes("import sklearn\n")) create(message="test1") # list all snapshots with no filters snapshot_list_1 = ls() assert snapshot_list_1 assert len(list(snapshot_list_1)) == 1 assert isinstance(snapshot_list_1[0], Snapshot) # Create a snapshot with default params, files, and environment test_filepath = os.path.join(self.temp_dir, "Dockerfile") with open(test_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) create(message="test2") # list all snapshots with no filters (works when more than 1 snapshot) snapshot_list_2 = ls() assert snapshot_list_2 assert len(list(snapshot_list_2)) == 2 assert isinstance(snapshot_list_2[0], Snapshot) assert isinstance(snapshot_list_2[1], Snapshot) # list snapshots with specific filter snapshot_list_3 = ls(filter="test2") assert snapshot_list_3 assert len(list(snapshot_list_3)) == 1 assert isinstance(snapshot_list_3[0], Snapshot) # list snapshots with filter of none snapshot_list_4 = ls(filter="test3") assert len(list(snapshot_list_4)) == 0
def test_create(self): # check project is not initialized if wrong home failed = False try: create(message="test", home=os.path.join("does", "not", "exist")) except InvalidProjectPathException: failed = True assert failed # Create a snapshot with default params # (fails w/ no commit) failed = False try: _ = create(message="test", home=self.temp_dir) except GitCommitDoesNotExist: failed = True assert failed # Create a snapshot with default params and files to commit test_filepath = os.path.join(self.temp_dir, "script.py") with open(test_filepath, "w") as f: f.write(to_unicode("import numpy\n")) f.write(to_unicode("import sklearn\n")) snapshot_obj_1 = create(message="test", home=self.temp_dir) assert snapshot_obj_1 assert isinstance(snapshot_obj_1, Snapshot) assert snapshot_obj_1.message == "test" assert snapshot_obj_1.code_id assert snapshot_obj_1.environment_id assert snapshot_obj_1.file_collection_id assert snapshot_obj_1.config == {} assert snapshot_obj_1.stats == {} # Create a snapshot with default params, files, and environment test_filepath = os.path.join(self.temp_dir, "Dockerfile") with open(test_filepath, "w") as f: f.write(to_unicode("FROM datmo/xgboost:cpu")) snapshot_obj_2 = create(message="test", home=self.temp_dir) assert snapshot_obj_2 assert isinstance(snapshot_obj_2, Snapshot) assert snapshot_obj_2.message == "test" assert snapshot_obj_2.code_id assert snapshot_obj_2.environment_id assert snapshot_obj_2.file_collection_id assert snapshot_obj_2.config == {} assert snapshot_obj_2.stats == {} assert snapshot_obj_2 != snapshot_obj_1
def test_create_from_task_fail_user_inputs(self): # Setup task # Create environment definition env_def_path = os.path.join(self.temp_dir, "Dockerfile") with open(env_def_path, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) task_obj = run("sh -c echo accuracy:0.45", home=self.temp_dir) # Test if failure if user gives other parameters failed = False try: _ = create(message="my test snapshot", task_id=task_obj.id, home=self.temp_dir, label="best", config={"foo": "bar"}, stats={"foo": "bar"}, commit_id="test_id") except SnapshotCreateFromTaskArgs: failed = True assert failed failed = False try: _ = create(message="my test snapshot", task_id=task_obj.id, home=self.temp_dir, label="best", config={"foo": "bar"}, stats={"foo": "bar"}, environment_id="test_id") except SnapshotCreateFromTaskArgs: failed = True assert failed failed = False try: _ = create(message="my test snapshot", task_id=task_obj.id, home=self.temp_dir, label="best", config={"foo": "bar"}, stats={"foo": "bar"}, filepaths=["mypath"]) except SnapshotCreateFromTaskArgs: failed = True assert failed
def test_delete(self): # check project is not initialized if wrong home Config().set_home(os.path.join("does", "not", "exist")) failed = False try: delete() except InvalidProjectPath: failed = True assert failed # check for snapshot id that does not exist Config().set_home(self.temp_dir) failed = False try: delete(snapshot_id="does_not_exist") except EntityNotFound: failed = True assert failed # delete a snapshot test_filepath = os.path.join(self.temp_dir, "script.py") with open(test_filepath, "wb") as f: f.write(to_bytes("import numpy\n")) f.write(to_bytes("import sklearn\n")) snapshot_obj = create(message="delete_test") snapshot_list_before_delete = ls(filter='delete_test') delete(snapshot_id=snapshot_obj.id) snapshot_list_after_delete = ls(filter='delete_test') assert len(snapshot_list_before_delete) == 1 assert len(snapshot_list_after_delete) == 0
def __setup(self): # Create a snapshot with default params and files to commit test_filepath = os.path.join(self.temp_dir, "script.py") with open(test_filepath, "wb") as f: f.write(to_bytes("import numpy\n")) f.write(to_bytes("import sklearn\n")) return create(message="test", paths=[test_filepath])
def test_create_from_task(self): # 1) Test if success with task files, results, and message # 2) Test if success with user given config and stats # TODO: test for failure case where tasks is not complete # Setup task # Create environment definition env_def_path = os.path.join(self.temp_dir, "Dockerfile") with open(env_def_path, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) task_controller = TaskController() task_obj = task_controller.create() task_obj = task_controller.run( task_obj.id, task_dict={"command": "sh -c echo accuracy:0.45"}) # 1) Test option 1 snapshot_obj = create( message="my test snapshot", run_id=task_obj.id, label="best", config={"foo": "bar"}) assert isinstance(snapshot_obj, Snapshot) assert snapshot_obj.message == "my test snapshot" assert snapshot_obj.label == "best" assert len(snapshot_obj.files) == 1 assert "task.log" in snapshot_obj.files[0].name assert snapshot_obj.config == {"foo": "bar"} assert snapshot_obj.stats == task_obj.results # Test option 2 snapshot_obj_2 = create( message="my test snapshot", run_id=task_obj.id, label="best", config={"foo": "bar"}, stats={"foo": "bar"}) assert isinstance(snapshot_obj, Snapshot) assert snapshot_obj_2.message == "my test snapshot" assert snapshot_obj_2.label == "best" assert len(snapshot_obj.files) == 1 assert "task.log" in snapshot_obj.files[0].name assert snapshot_obj_2.config == {"foo": "bar"} assert snapshot_obj_2.stats == {"foo": "bar"}
def test_create_from_task(self): # 1) Test if success with task files, results, and message # 2) Test if success with user given config and stats # TODO: test for failure case where tasks is not complete # Setup task # Create environment definition env_def_path = os.path.join(self.temp_dir, "Dockerfile") with open(env_def_path, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) task_obj = run("sh -c echo accuracy:0.45", home=self.temp_dir) # 1) Test option 1 snapshot_obj = create(message="my test snapshot", task_id=task_obj.id, home=self.temp_dir, label="best", config={"foo": "bar"}) assert isinstance(snapshot_obj, Snapshot) assert snapshot_obj.message == "my test snapshot" assert snapshot_obj.label == "best" assert snapshot_obj.config == {"foo": "bar"} assert snapshot_obj.stats == task_obj.results # Test option 2 snapshot_obj_2 = create(message="my test snapshot", task_id=task_obj.id, home=self.temp_dir, label="best", config={"foo": "bar"}, stats={"foo": "bar"}) assert isinstance(snapshot_obj, Snapshot) assert snapshot_obj_2.message == "my test snapshot" assert snapshot_obj_2.label == "best" assert snapshot_obj_2.config == {"foo": "bar"} assert snapshot_obj_2.stats == {"foo": "bar"}
def test_create(self): # check project is not initialized if wrong home Config().set_home(os.path.join("does", "not", "exist")) failed = False try: create(message="test") except InvalidProjectPath: failed = True assert failed # Create a snapshot with default params # (pass w/ no commit) Config().set_home(self.temp_dir) result = create(message="test") assert result # Create a snapshot with default params and files to commit test_filepath = os.path.join(self.temp_dir, "script.py") with open(test_filepath, "wb") as f: f.write(to_bytes("import numpy\n")) f.write(to_bytes("import sklearn\n")) snapshot_obj_1 = create(message="test") assert snapshot_obj_1 assert isinstance(snapshot_obj_1, Snapshot) assert snapshot_obj_1.message == "test" assert snapshot_obj_1.code_id assert snapshot_obj_1.environment_id assert snapshot_obj_1.files == [] assert snapshot_obj_1.config == {} assert snapshot_obj_1.stats == {} # Create a snapshot with default params, files, and environment test_filepath = os.path.join(self.temp_dir, "Dockerfile") with open(test_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) snapshot_obj_2 = create(message="test") assert snapshot_obj_2 assert isinstance(snapshot_obj_2, Snapshot) assert snapshot_obj_2.message == "test" assert snapshot_obj_2.code_id assert snapshot_obj_2.environment_id assert snapshot_obj_2.files == [] assert snapshot_obj_2.config == {} assert snapshot_obj_2.stats == {} assert snapshot_obj_2 != snapshot_obj_1 # Create a snapshot with default params, files, and environment being passed in test_filepath = os.path.join(self.temp_dir, "Dockerfile") with open(test_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) snapshot_obj_3 = create(message="test", env=test_filepath) assert snapshot_obj_3 assert isinstance(snapshot_obj_3, Snapshot) assert snapshot_obj_3.message == "test" assert snapshot_obj_3.code_id assert snapshot_obj_3.environment_id assert snapshot_obj_3.files == [] assert snapshot_obj_3.config == {} assert snapshot_obj_3.stats == {} assert snapshot_obj_3 != snapshot_obj_1
def test_update(self): # check project is not initialized if wrong home Config().set_home(os.path.join("does", "not", "exist")) failed = False try: delete() except InvalidProjectPath: failed = True assert failed # check for snapshot id that does not exist Config().set_home(self.temp_dir) failed = False try: update(snapshot_id="does_not_exist") except EntityNotFound: failed = True assert failed test_config = {"config_foo": "bar"} test_stats = {"stats_foo": "bar"} test_message = "new_message" test_label = "new_label" # update both message and label snapshot_entity = self.__setup() updated_snapshot_obj = update(snapshot_id=snapshot_entity.id, message=test_message, label=test_label) assert updated_snapshot_obj.id == snapshot_entity.id assert updated_snapshot_obj.message == test_message assert updated_snapshot_obj.label == test_label # update message snapshot_obj_1 = create(message="test") updated_snapshot_obj_1 = update(snapshot_id=snapshot_obj_1.id, message=test_message) assert updated_snapshot_obj_1.id == snapshot_obj_1.id assert updated_snapshot_obj_1.message == test_message # update label snapshot_obj_2 = create(message="test") updated_snapshot_obj_2 = update(snapshot_id=snapshot_obj_2.id, label=test_label) assert updated_snapshot_obj_2.id == snapshot_obj_2.id assert updated_snapshot_obj_2.message == test_message # test config snapshot_obj_3 = create(message="test") updated_snapshot_obj_3 = update(snapshot_id=snapshot_obj_3.id, config=test_config) assert updated_snapshot_obj_3.id == snapshot_obj_3.id assert updated_snapshot_obj_3.config == test_config # test stats snapshot_obj_4 = create(message="test") updated_snapshot_obj_4 = update(snapshot_id=snapshot_obj_4.id, stats=test_stats) assert updated_snapshot_obj_4.id == snapshot_obj_4.id assert updated_snapshot_obj_4.stats == test_stats