def _update_ui_url( self, db: DBInterface, db_session: Session, project: str, uid: str, crd_object, run: Dict = None, leader_session: Optional[str] = None, ): app_state = (crd_object.get("status", {}).get("applicationState", {}).get("state")) state = SparkApplicationStates.spark_application_state_to_run_state( app_state) ui_url = None if state == RunStates.running: ui_url = (crd_object.get("status", {}).get("driverInfo", {}).get("webUIIngressAddress")) db_ui_url = run.get("status", {}).get("ui_url") if db_ui_url == ui_url: return run.setdefault("status", {})["ui_url"] = ui_url db.store_run(db_session, run, uid, project, leader_session=leader_session)
def test_list_runs_name_filter(db: DBInterface, db_session: Session): project = "project" run_name_1 = "run_name_1" run_name_2 = "run_name_2" run_1 = {"metadata": {"name": run_name_1}, "status": {"bla": "blabla"}} run_2 = {"metadata": {"name": run_name_2}, "status": {"bla": "blabla"}} # run with no name run_3 = {"metadata": {}, "status": {"bla": "blabla"}} run_uid_1 = "run_uid_1" run_uid_2 = "run_uid_2" run_uid_3 = "run_uid_3" db.store_run(db_session, run_1, run_uid_1, project) db.store_run(db_session, run_2, run_uid_2, project) db.store_run(db_session, run_3, run_uid_3, project) runs = db.list_runs(db_session, project=project) assert len(runs) == 3 runs = db.list_runs(db_session, name=run_name_1, project=project) assert len(runs) == 1 assert runs[0]["metadata"]["name"] == run_name_1 runs = db.list_runs(db_session, name=run_name_2, project=project) assert len(runs) == 1 assert runs[0]["metadata"]["name"] == run_name_2 runs = db.list_runs(db_session, name="~RUN_naMe", project=project) assert len(runs) == 2
def test_list_runs_name_filter(db: DBInterface, db_session: Session): run_name_1 = "run_name_1" run_name_2 = "run_name_2" run_1 = {"metadata": {"name": run_name_1}, "status": {"bla": "blabla"}} run_2 = {"metadata": {"name": run_name_2}, "status": {"bla": "blabla"}} run_uid_1 = "run_uid_1" run_uid_2 = "run_uid_2" db.store_run( db_session, run_1, run_uid_1, ) db.store_run( db_session, run_2, run_uid_2, ) runs = db.list_runs(db_session) assert len(runs) == 2 runs = db.list_runs(db_session, name=run_name_1) assert len(runs) == 1 assert runs[0]["metadata"]["name"] == run_name_1 runs = db.list_runs(db_session, name=run_name_2) assert len(runs) == 1 assert runs[0]["metadata"]["name"] == run_name_2 runs = db.list_runs(db_session, name="run_name") assert len(runs) == 2
def test_list_runs_state_filter(db: DBInterface, db_session: Session): run_without_state_uid = "run_without_state_uid" run_without_state = { "metadata": { "uid": run_without_state_uid }, "bla": "blabla" } db.store_run( db_session, run_without_state, run_without_state_uid, ) run_with_json_state_state = "some_json_state" run_with_json_state_uid = "run_with_json_state_uid" run_with_json_state = { "metadata": { "uid": run_with_json_state_uid }, "status": { "state": run_with_json_state_state }, } run = Run( uid=run_with_json_state_uid, project=config.default_project, iteration=0, start_time=datetime.now(timezone.utc), ) run.struct = run_with_json_state db._upsert(db_session, run, ignore=True) run_with_record_state_state = "some_record_state" run_with_record_state_uid = "run_with_record_state_uid" run_with_record_state = { "metadata": { "uid": run_with_record_state_uid }, "bla": "blabla", } run = Run( uid=run_with_record_state_uid, project=config.default_project, iteration=0, state=run_with_record_state_state, start_time=datetime.now(timezone.utc), ) run.struct = run_with_record_state db._upsert(db_session, run, ignore=True) run_with_equal_json_and_record_state_state = "some_equal_json_and_record_state" run_with_equal_json_and_record_state_uid = ( "run_with_equal_json_and_record_state_uid") run_with_equal_json_and_record_state = { "metadata": { "uid": run_with_equal_json_and_record_state_uid }, "status": { "state": run_with_equal_json_and_record_state_state }, } db.store_run( db_session, run_with_equal_json_and_record_state, run_with_equal_json_and_record_state_uid, ) run_with_unequal_json_and_record_state_json_state = "some_unequal_json_state" run_with_unequal_json_and_record_state_record_state = "some_unequal_record_state" run_with_unequal_json_and_record_state_uid = ( "run_with_unequal_json_and_record_state_uid") run_with_unequal_json_and_record_state = { "metadata": { "uid": run_with_unequal_json_and_record_state_uid }, "status": { "state": run_with_unequal_json_and_record_state_json_state }, } run = Run( uid=run_with_unequal_json_and_record_state_uid, project=config.default_project, iteration=0, state=run_with_unequal_json_and_record_state_record_state, start_time=datetime.now(timezone.utc), ) run.struct = run_with_unequal_json_and_record_state db._upsert(db_session, run, ignore=True) runs = db.list_runs(db_session) assert len(runs) == 5 runs = db.list_runs(db_session, state="some_") assert len(runs) == 4 assert run_without_state_uid not in [ run["metadata"]["uid"] for run in runs ] runs = db.list_runs(db_session, state=run_with_json_state_state) assert len(runs) == 1 assert runs[0]["metadata"]["uid"] == run_with_json_state_uid runs = db.list_runs(db_session, state=run_with_record_state_state) assert len(runs) == 1 assert runs[0]["metadata"]["uid"] == run_with_record_state_uid runs = db.list_runs(db_session, state=run_with_equal_json_and_record_state_state) assert len(runs) == 1 assert runs[0]["metadata"][ "uid"] == run_with_equal_json_and_record_state_uid runs = db.list_runs( db_session, state=run_with_unequal_json_and_record_state_json_state) assert len(runs) == 1 assert runs[0]["metadata"][ "uid"] == run_with_unequal_json_and_record_state_uid runs = db.list_runs( db_session, state=run_with_unequal_json_and_record_state_record_state) assert len(runs) == 0
def _create_resources_of_all_kinds(db: DBInterface, db_session: Session, project: str): # Create several functions with several tags labels = { "name": "value", "name2": "value2", } function = { "bla": "blabla", "metadata": { "labels": labels }, "status": { "bla": "blabla" }, } function_names = ["function_name_1", "function_name_2", "function_name_3"] function_tags = ["some_tag", "some_tag2", "some_tag3"] for function_name in function_names: for function_tag in function_tags: db.store_function( db_session, function, function_name, project, tag=function_tag, versioned=True, ) # Create several artifacts with several tags artifact = { "bla": "blabla", "labels": labels, "status": { "bla": "blabla" }, } artifact_keys = ["artifact_key_1", "artifact_key_2", "artifact_key_3"] artifact_uids = ["some_uid", "some_uid2", "some_uid3"] artifact_tags = ["some_tag", "some_tag2", "some_tag3"] for artifact_key in artifact_keys: for artifact_uid in artifact_uids: for artifact_tag in artifact_tags: for artifact_iter in range(3): db.store_artifact( db_session, artifact_key, artifact, artifact_uid, artifact_iter, artifact_tag, project, ) # Create several runs run = { "bla": "blabla", "metadata": { "labels": labels }, "status": { "bla": "blabla" }, } run_uids = ["some_uid", "some_uid2", "some_uid3"] for run_uid in run_uids: for run_iter in range(3): db.store_run(db_session, run, run_uid, project, run_iter) # Create several logs log = b"some random log" log_uids = ["some_uid", "some_uid2", "some_uid3"] for log_uid in log_uids: db.store_log(db_session, log_uid, project, log) # Create several schedule schedule = { "bla": "blabla", "status": { "bla": "blabla" }, } schedule_cron_trigger = schemas.ScheduleCronTrigger(year=1999) schedule_names = ["schedule_name_1", "schedule_name_2", "schedule_name_3"] for schedule_name in schedule_names: db.create_schedule( db_session, project, schedule_name, schemas.ScheduleKinds.job, schedule, schedule_cron_trigger, labels, ) feature_set = schemas.FeatureSet( metadata=schemas.ObjectMetadata(name="dummy", tag="latest", labels={"owner": "nobody"}), spec=schemas.FeatureSetSpec( entities=[ schemas.Entity(name="ent1", value_type="str", labels={"label": "1"}) ], features=[ schemas.Feature(name="feat1", value_type="str", labels={"label": "1"}) ], ), status={}, ) db.create_feature_set(db_session, project, feature_set)