def _update_ui_url(
self,
db: DBInterface,
db_session: Session,
project: str,
uid: str,
crd_object,
run: Dict = None,
leader_session: Optional[str] = None,
):
app_state = (crd_object.get("status", {}).get("applicationState",
{}).get("state"))
state = SparkApplicationStates.spark_application_state_to_run_state(
app_state)
ui_url = None
if state == RunStates.running:
ui_url = (crd_object.get("status",
{}).get("driverInfo",
{}).get("webUIIngressAddress"))
db_ui_url = run.get("status", {}).get("ui_url")
if db_ui_url == ui_url:
return
run.setdefault("status", {})["ui_url"] = ui_url
db.store_run(db_session,
run,
uid,
project,
leader_session=leader_session)
def test_list_runs_name_filter(db: DBInterface, db_session: Session):
project = "project"
run_name_1 = "run_name_1"
run_name_2 = "run_name_2"
run_1 = {"metadata": {"name": run_name_1}, "status": {"bla": "blabla"}}
run_2 = {"metadata": {"name": run_name_2}, "status": {"bla": "blabla"}}
# run with no name
run_3 = {"metadata": {}, "status": {"bla": "blabla"}}
run_uid_1 = "run_uid_1"
run_uid_2 = "run_uid_2"
run_uid_3 = "run_uid_3"
db.store_run(db_session, run_1, run_uid_1, project)
db.store_run(db_session, run_2, run_uid_2, project)
db.store_run(db_session, run_3, run_uid_3, project)
runs = db.list_runs(db_session, project=project)
assert len(runs) == 3
runs = db.list_runs(db_session, name=run_name_1, project=project)
assert len(runs) == 1
assert runs[0]["metadata"]["name"] == run_name_1
runs = db.list_runs(db_session, name=run_name_2, project=project)
assert len(runs) == 1
assert runs[0]["metadata"]["name"] == run_name_2
runs = db.list_runs(db_session, name="~RUN_naMe", project=project)
assert len(runs) == 2
def test_list_runs_name_filter(db: DBInterface, db_session: Session):
run_name_1 = "run_name_1"
run_name_2 = "run_name_2"
run_1 = {"metadata": {"name": run_name_1}, "status": {"bla": "blabla"}}
run_2 = {"metadata": {"name": run_name_2}, "status": {"bla": "blabla"}}
run_uid_1 = "run_uid_1"
run_uid_2 = "run_uid_2"
db.store_run(
db_session,
run_1,
run_uid_1,
)
db.store_run(
db_session,
run_2,
run_uid_2,
)
runs = db.list_runs(db_session)
assert len(runs) == 2
runs = db.list_runs(db_session, name=run_name_1)
assert len(runs) == 1
assert runs[0]["metadata"]["name"] == run_name_1
runs = db.list_runs(db_session, name=run_name_2)
assert len(runs) == 1
assert runs[0]["metadata"]["name"] == run_name_2
runs = db.list_runs(db_session, name="run_name")
assert len(runs) == 2
def test_list_runs_state_filter(db: DBInterface, db_session: Session):
run_without_state_uid = "run_without_state_uid"
run_without_state = {
"metadata": {
"uid": run_without_state_uid
},
"bla": "blabla"
}
db.store_run(
db_session,
run_without_state,
run_without_state_uid,
)
run_with_json_state_state = "some_json_state"
run_with_json_state_uid = "run_with_json_state_uid"
run_with_json_state = {
"metadata": {
"uid": run_with_json_state_uid
},
"status": {
"state": run_with_json_state_state
},
}
run = Run(
uid=run_with_json_state_uid,
project=config.default_project,
iteration=0,
start_time=datetime.now(timezone.utc),
)
run.struct = run_with_json_state
db._upsert(db_session, run, ignore=True)
run_with_record_state_state = "some_record_state"
run_with_record_state_uid = "run_with_record_state_uid"
run_with_record_state = {
"metadata": {
"uid": run_with_record_state_uid
},
"bla": "blabla",
}
run = Run(
uid=run_with_record_state_uid,
project=config.default_project,
iteration=0,
state=run_with_record_state_state,
start_time=datetime.now(timezone.utc),
)
run.struct = run_with_record_state
db._upsert(db_session, run, ignore=True)
run_with_equal_json_and_record_state_state = "some_equal_json_and_record_state"
run_with_equal_json_and_record_state_uid = (
"run_with_equal_json_and_record_state_uid")
run_with_equal_json_and_record_state = {
"metadata": {
"uid": run_with_equal_json_and_record_state_uid
},
"status": {
"state": run_with_equal_json_and_record_state_state
},
}
db.store_run(
db_session,
run_with_equal_json_and_record_state,
run_with_equal_json_and_record_state_uid,
)
run_with_unequal_json_and_record_state_json_state = "some_unequal_json_state"
run_with_unequal_json_and_record_state_record_state = "some_unequal_record_state"
run_with_unequal_json_and_record_state_uid = (
"run_with_unequal_json_and_record_state_uid")
run_with_unequal_json_and_record_state = {
"metadata": {
"uid": run_with_unequal_json_and_record_state_uid
},
"status": {
"state": run_with_unequal_json_and_record_state_json_state
},
}
run = Run(
uid=run_with_unequal_json_and_record_state_uid,
project=config.default_project,
iteration=0,
state=run_with_unequal_json_and_record_state_record_state,
start_time=datetime.now(timezone.utc),
)
run.struct = run_with_unequal_json_and_record_state
db._upsert(db_session, run, ignore=True)
runs = db.list_runs(db_session)
assert len(runs) == 5
runs = db.list_runs(db_session, state="some_")
assert len(runs) == 4
assert run_without_state_uid not in [
run["metadata"]["uid"] for run in runs
]
runs = db.list_runs(db_session, state=run_with_json_state_state)
assert len(runs) == 1
assert runs[0]["metadata"]["uid"] == run_with_json_state_uid
runs = db.list_runs(db_session, state=run_with_record_state_state)
assert len(runs) == 1
assert runs[0]["metadata"]["uid"] == run_with_record_state_uid
runs = db.list_runs(db_session,
state=run_with_equal_json_and_record_state_state)
assert len(runs) == 1
assert runs[0]["metadata"][
"uid"] == run_with_equal_json_and_record_state_uid
runs = db.list_runs(
db_session, state=run_with_unequal_json_and_record_state_json_state)
assert len(runs) == 1
assert runs[0]["metadata"][
"uid"] == run_with_unequal_json_and_record_state_uid
runs = db.list_runs(
db_session, state=run_with_unequal_json_and_record_state_record_state)
assert len(runs) == 0
def _create_resources_of_all_kinds(db: DBInterface, db_session: Session,
project: str):
# Create several functions with several tags
labels = {
"name": "value",
"name2": "value2",
}
function = {
"bla": "blabla",
"metadata": {
"labels": labels
},
"status": {
"bla": "blabla"
},
}
function_names = ["function_name_1", "function_name_2", "function_name_3"]
function_tags = ["some_tag", "some_tag2", "some_tag3"]
for function_name in function_names:
for function_tag in function_tags:
db.store_function(
db_session,
function,
function_name,
project,
tag=function_tag,
versioned=True,
)
# Create several artifacts with several tags
artifact = {
"bla": "blabla",
"labels": labels,
"status": {
"bla": "blabla"
},
}
artifact_keys = ["artifact_key_1", "artifact_key_2", "artifact_key_3"]
artifact_uids = ["some_uid", "some_uid2", "some_uid3"]
artifact_tags = ["some_tag", "some_tag2", "some_tag3"]
for artifact_key in artifact_keys:
for artifact_uid in artifact_uids:
for artifact_tag in artifact_tags:
for artifact_iter in range(3):
db.store_artifact(
db_session,
artifact_key,
artifact,
artifact_uid,
artifact_iter,
artifact_tag,
project,
)
# Create several runs
run = {
"bla": "blabla",
"metadata": {
"labels": labels
},
"status": {
"bla": "blabla"
},
}
run_uids = ["some_uid", "some_uid2", "some_uid3"]
for run_uid in run_uids:
for run_iter in range(3):
db.store_run(db_session, run, run_uid, project, run_iter)
# Create several logs
log = b"some random log"
log_uids = ["some_uid", "some_uid2", "some_uid3"]
for log_uid in log_uids:
db.store_log(db_session, log_uid, project, log)
# Create several schedule
schedule = {
"bla": "blabla",
"status": {
"bla": "blabla"
},
}
schedule_cron_trigger = schemas.ScheduleCronTrigger(year=1999)
schedule_names = ["schedule_name_1", "schedule_name_2", "schedule_name_3"]
for schedule_name in schedule_names:
db.create_schedule(
db_session,
project,
schedule_name,
schemas.ScheduleKinds.job,
schedule,
schedule_cron_trigger,
labels,
)
feature_set = schemas.FeatureSet(
metadata=schemas.ObjectMetadata(name="dummy",
tag="latest",
labels={"owner": "nobody"}),
spec=schemas.FeatureSetSpec(
entities=[
schemas.Entity(name="ent1",
value_type="str",
labels={"label": "1"})
],
features=[
schemas.Feature(name="feat1",
value_type="str",
labels={"label": "1"})
],
),
status={},
)
db.create_feature_set(db_session, project, feature_set)
