def get_table_query_examples(
table_id,
environment_id,
uid=None,
engine_id=None,
with_table_id=None,
limit=10,
offset=0,
):
api_assert(limit < 100)
with DBSession() as session:
verify_environment_permission([environment_id])
verify_data_table_permission(table_id, session=session)
engines = admin_logic.get_query_engines_by_environment(environment_id,
session=session)
engine_ids = [engine.id for engine in engines]
api_assert(engine_id is None or engine_id in engine_ids,
"Invalid engine id")
query_logs = logic.get_table_query_examples(
table_id,
engine_ids,
uid=uid,
engine_id=engine_id,
with_table_id=with_table_id,
limit=limit,
offset=offset,
session=session,
)
query_ids = [log.query_execution_id for log in query_logs]
return query_ids
def remove_table_ownership(table_id):
"""Remove a table ownership"""
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return logic.delete_table_ownership(table_id=table_id,
uid=current_user.id,
session=session)
def get_table_query_examples_engines(table_id, environment_id):
verify_environment_permission([environment_id])
verify_data_table_permission(table_id)
engines = logic.get_query_example_engines(table_id,
environment_id=environment_id)
return [{"engine_id": r[0], "count": r[1]} for r in engines]
def get_all_table_ownerships_by_table_id(table_id):
"""Add all table ownerships"""
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return logic.get_all_table_ownerships_by_table_id(table_id=table_id,
session=session)
def create_tag_item(table_id, tag):
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return logic.create_tag_item(table_id=table_id,
tag_name=tag,
uid=current_user.id,
session=session)
def get_table_column_stats(column_id):
"""Get all table stats column by id"""
with DBSession() as session:
column = logic.get_column_by_id(column_id, session=session)
verify_data_table_permission(column.table_id, session=session)
return DataTableColumnStatistics.get_all(column_id=column_id,
session=session)
def create_table_samples(
table_id,
environment_id,
engine_id,
partition=None,
where=None,
order_by=None,
order_by_asc=True,
limit=100,
):
with DBSession() as session:
api_assert(limit <= 100, "Too many rows requested")
verify_environment_permission([environment_id])
verify_data_table_permission(table_id, session=session)
verify_query_engine_environment_permission(engine_id,
environment_id,
session=session)
task = run_sample_query.apply_async(args=[
table_id,
engine_id,
current_user.id,
limit,
partition,
where,
order_by,
order_by_asc,
])
return task.task_id
def get_column(column_id, with_table=False):
with DBSession() as session:
column = logic.get_column_by_id(column_id, session=session)
verify_data_table_permission(column.table_id, session=session)
column_dict = column.to_dict(with_table)
return column_dict
def delete_tag_from_table(table_id: int, tag_name: str):
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return logic.delete_tag_from_table(
table_id=table_id,
tag_name=tag_name,
user_is_admin=current_user.is_admin,
session=session,
)
def get_table_query_examples_users(table_id, environment_id, limit=5):
api_assert(limit <= 10)
verify_environment_permission([environment_id])
verify_data_table_permission(table_id)
engines = admin_logic.get_query_engines_by_environment(environment_id)
engine_ids = [engine.id for engine in engines]
users = logic.get_query_example_users(table_id, engine_ids, limit=limit)
return [{"uid": r[0], "count": r[1]} for r in users]
def add_tag_to_table(table_id, tag):
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return logic.add_tag_to_table(
table_id=table_id,
tag_name=tag,
uid=current_user.id,
user_is_admin=current_user.is_admin,
session=session,
)
def update_table_boost_score(data):
"""Batch update table boost scores"""
# TODO: verify user is a service account
with DBSession() as session:
for d in data:
verify_data_table_permission(d["table_id"], session=session)
logic.update_table(id=d["table_id"],
score=d["boost_score"],
session=session)
return
def get_table_samples(table_id, environment_id):
try:
with DBSession() as session:
verify_environment_permission([environment_id])
verify_data_table_permission(table_id, session=session)
return mysql_cache.get_key(
f"table_samples_{table_id}_{current_user.id}",
expires_after=seconds_in_a_day,
session=session,
)
except LookupError:
return None
def update_table_warning(warning_id, **fields):
warning = DataTableWarning.get(id=warning_id)
verify_data_table_permission(warning.table_id)
if "severity" in fields:
fields["severity"] = DataTableWarningSeverity(fields["severity"])
return DataTableWarning.update(
id=warning_id,
fields={
**fields, "updated_by": current_user.id
},
field_names=["message", "severity", "updated_by"],
)
def create_table_warning(table_id, message, severity):
verify_data_table_permission(table_id)
return DataTableWarning.create({
"message":
message,
"severity":
DataTableWarningSeverity(severity),
"created_by":
current_user.id,
"updated_by":
current_user.id,
"table_id":
table_id,
})
def update_table(table_id, description=None, golden=None):
"""Update a table"""
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
if description:
logic.update_table_information(table_id,
description=description,
session=session)
if golden is not None:
api_assert(current_user.is_admin,
"Golden table can only be updated by Admin")
logic.update_table(table_id, golden=golden, session=session)
return logic.get_table_by_id(table_id, session=session)
def create_table_stats(data):
"""Batch add/update table stats"""
# TODO: verify user is a service account
with DBSession() as session:
for d in data:
verify_data_table_permission(d["table_id"], session=session)
for s in d["stats"]:
logic.upsert_table_stat(
table_id=d["table_id"],
key=s["key"],
value=s["value"],
uid=current_user.id,
session=session,
)
return
def refresh_table_from_metastore(table_id):
"""Refetch table info from metastore"""
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
table = logic.get_table_by_id(table_id, session=session)
schema = table.data_schema
metastore_id = schema.metastore_id
metastore_loader = get_metastore_loader(metastore_id, session=session)
metastore_loader.sync_create_or_update_table(schema.name,
table.name,
session=session)
session.refresh(table)
return table
def create_table_column_stats(data):
"""Batch add/update table column stats"""
# TODO: verify user is a service account
with DBSession() as session:
for d in data:
column = logic.get_column_by_id(d["column_id"], session=session)
if column:
verify_data_table_permission(column.table_id, session=session)
for s in d["stats"]:
logic.upsert_table_column_stat(
column_id=d["column_id"],
key=s["key"],
value=s["value"],
uid=current_user.id,
session=session,
)
return
def get_table_samples_raw_query(
table_id,
partition=None,
where=None,
order_by=None,
order_by_asc=True,
limit=100,
):
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return make_samples_query(
table_id,
limit=limit,
partition=partition,
where=where,
order_by=order_by,
order_by_asc=order_by_asc,
session=session,
)
def delete_table_warning(warning_id):
warning = DataTableWarning.get(id=warning_id)
verify_data_table_permission(warning.table_id)
DataTableWarning.delete(warning_id)
def get_tag_items(table_id):
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return logic.get_tag_items_by_table_id(table_id=table_id,
session=session)
def get_table_stats(table_id):
"""Get all table stats by id"""
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return DataTableStatistics.get_all(table_id=table_id, session=session)
def get_table_query_examples_concurrences(table_id, limit=5):
api_assert(limit <= 10)
verify_data_table_permission(table_id)
concurrences = logic.get_query_example_concurrences(table_id, limit=limit)
return [{"table_id": r[0], "count": r[1]} for r in concurrences]
def delete_tag_item(table_id, tag_item_id):
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return logic.delete_tag_item(tag_item_id=tag_item_id, session=session)
def get_columns_from_table(table_id):
with DBSession() as session:
verify_data_table_permission(table_id, session=session)
return logic.get_column_by_table_id(table_id, session=session)
