def get_table_query_examples( table_id, environment_id, uid=None, engine_id=None, with_table_id=None, limit=10, offset=0, ): api_assert(limit < 100) with DBSession() as session: verify_environment_permission([environment_id]) verify_data_table_permission(table_id, session=session) engines = admin_logic.get_query_engines_by_environment(environment_id, session=session) engine_ids = [engine.id for engine in engines] api_assert(engine_id is None or engine_id in engine_ids, "Invalid engine id") query_logs = logic.get_table_query_examples( table_id, engine_ids, uid=uid, engine_id=engine_id, with_table_id=with_table_id, limit=limit, offset=offset, session=session, ) query_ids = [log.query_execution_id for log in query_logs] return query_ids
def remove_table_ownership(table_id): """Remove a table ownership""" with DBSession() as session: verify_data_table_permission(table_id, session=session) return logic.delete_table_ownership(table_id=table_id, uid=current_user.id, session=session)
def get_table_query_examples_engines(table_id, environment_id): verify_environment_permission([environment_id]) verify_data_table_permission(table_id) engines = logic.get_query_example_engines(table_id, environment_id=environment_id) return [{"engine_id": r[0], "count": r[1]} for r in engines]
def get_all_table_ownerships_by_table_id(table_id): """Add all table ownerships""" with DBSession() as session: verify_data_table_permission(table_id, session=session) return logic.get_all_table_ownerships_by_table_id(table_id=table_id, session=session)
def create_tag_item(table_id, tag): with DBSession() as session: verify_data_table_permission(table_id, session=session) return logic.create_tag_item(table_id=table_id, tag_name=tag, uid=current_user.id, session=session)
def get_table_column_stats(column_id): """Get all table stats column by id""" with DBSession() as session: column = logic.get_column_by_id(column_id, session=session) verify_data_table_permission(column.table_id, session=session) return DataTableColumnStatistics.get_all(column_id=column_id, session=session)
def create_table_samples( table_id, environment_id, engine_id, partition=None, where=None, order_by=None, order_by_asc=True, limit=100, ): with DBSession() as session: api_assert(limit <= 100, "Too many rows requested") verify_environment_permission([environment_id]) verify_data_table_permission(table_id, session=session) verify_query_engine_environment_permission(engine_id, environment_id, session=session) task = run_sample_query.apply_async(args=[ table_id, engine_id, current_user.id, limit, partition, where, order_by, order_by_asc, ]) return task.task_id
def get_column(column_id, with_table=False): with DBSession() as session: column = logic.get_column_by_id(column_id, session=session) verify_data_table_permission(column.table_id, session=session) column_dict = column.to_dict(with_table) return column_dict
def delete_tag_from_table(table_id: int, tag_name: str): with DBSession() as session: verify_data_table_permission(table_id, session=session) return logic.delete_tag_from_table( table_id=table_id, tag_name=tag_name, user_is_admin=current_user.is_admin, session=session, )
def get_table_query_examples_users(table_id, environment_id, limit=5): api_assert(limit <= 10) verify_environment_permission([environment_id]) verify_data_table_permission(table_id) engines = admin_logic.get_query_engines_by_environment(environment_id) engine_ids = [engine.id for engine in engines] users = logic.get_query_example_users(table_id, engine_ids, limit=limit) return [{"uid": r[0], "count": r[1]} for r in users]
def add_tag_to_table(table_id, tag): with DBSession() as session: verify_data_table_permission(table_id, session=session) return logic.add_tag_to_table( table_id=table_id, tag_name=tag, uid=current_user.id, user_is_admin=current_user.is_admin, session=session, )
def update_table_boost_score(data): """Batch update table boost scores""" # TODO: verify user is a service account with DBSession() as session: for d in data: verify_data_table_permission(d["table_id"], session=session) logic.update_table(id=d["table_id"], score=d["boost_score"], session=session) return
def get_table_samples(table_id, environment_id): try: with DBSession() as session: verify_environment_permission([environment_id]) verify_data_table_permission(table_id, session=session) return mysql_cache.get_key( f"table_samples_{table_id}_{current_user.id}", expires_after=seconds_in_a_day, session=session, ) except LookupError: return None
def update_table_warning(warning_id, **fields): warning = DataTableWarning.get(id=warning_id) verify_data_table_permission(warning.table_id) if "severity" in fields: fields["severity"] = DataTableWarningSeverity(fields["severity"]) return DataTableWarning.update( id=warning_id, fields={ **fields, "updated_by": current_user.id }, field_names=["message", "severity", "updated_by"], )
def create_table_warning(table_id, message, severity): verify_data_table_permission(table_id) return DataTableWarning.create({ "message": message, "severity": DataTableWarningSeverity(severity), "created_by": current_user.id, "updated_by": current_user.id, "table_id": table_id, })
def update_table(table_id, description=None, golden=None): """Update a table""" with DBSession() as session: verify_data_table_permission(table_id, session=session) if description: logic.update_table_information(table_id, description=description, session=session) if golden is not None: api_assert(current_user.is_admin, "Golden table can only be updated by Admin") logic.update_table(table_id, golden=golden, session=session) return logic.get_table_by_id(table_id, session=session)
def create_table_stats(data): """Batch add/update table stats""" # TODO: verify user is a service account with DBSession() as session: for d in data: verify_data_table_permission(d["table_id"], session=session) for s in d["stats"]: logic.upsert_table_stat( table_id=d["table_id"], key=s["key"], value=s["value"], uid=current_user.id, session=session, ) return
def refresh_table_from_metastore(table_id): """Refetch table info from metastore""" with DBSession() as session: verify_data_table_permission(table_id, session=session) table = logic.get_table_by_id(table_id, session=session) schema = table.data_schema metastore_id = schema.metastore_id metastore_loader = get_metastore_loader(metastore_id, session=session) metastore_loader.sync_create_or_update_table(schema.name, table.name, session=session) session.refresh(table) return table
def create_table_column_stats(data): """Batch add/update table column stats""" # TODO: verify user is a service account with DBSession() as session: for d in data: column = logic.get_column_by_id(d["column_id"], session=session) if column: verify_data_table_permission(column.table_id, session=session) for s in d["stats"]: logic.upsert_table_column_stat( column_id=d["column_id"], key=s["key"], value=s["value"], uid=current_user.id, session=session, ) return
def get_table_samples_raw_query( table_id, partition=None, where=None, order_by=None, order_by_asc=True, limit=100, ): with DBSession() as session: verify_data_table_permission(table_id, session=session) return make_samples_query( table_id, limit=limit, partition=partition, where=where, order_by=order_by, order_by_asc=order_by_asc, session=session, )
def delete_table_warning(warning_id): warning = DataTableWarning.get(id=warning_id) verify_data_table_permission(warning.table_id) DataTableWarning.delete(warning_id)
def get_tag_items(table_id): with DBSession() as session: verify_data_table_permission(table_id, session=session) return logic.get_tag_items_by_table_id(table_id=table_id, session=session)
def get_table_stats(table_id): """Get all table stats by id""" with DBSession() as session: verify_data_table_permission(table_id, session=session) return DataTableStatistics.get_all(table_id=table_id, session=session)
def get_table_query_examples_concurrences(table_id, limit=5): api_assert(limit <= 10) verify_data_table_permission(table_id) concurrences = logic.get_query_example_concurrences(table_id, limit=limit) return [{"table_id": r[0], "count": r[1]} for r in concurrences]
def delete_tag_item(table_id, tag_item_id): with DBSession() as session: verify_data_table_permission(table_id, session=session) return logic.delete_tag_item(tag_item_id=tag_item_id, session=session)
def get_columns_from_table(table_id): with DBSession() as session: verify_data_table_permission(table_id, session=session) return logic.get_column_by_table_id(table_id, session=session)