def get_table_query_examples_engines(table_id, environment_id):
verify_environment_permission([environment_id])
verify_data_table_permission(table_id)
engines = logic.get_query_example_engines(table_id,
environment_id=environment_id)
return [{"engine_id": r[0], "count": r[1]} for r in engines]
def get_table_query_examples(
table_id,
environment_id,
uid=None,
engine_id=None,
with_table_id=None,
limit=10,
offset=0,
):
api_assert(limit < 100)
with DBSession() as session:
verify_environment_permission([environment_id])
verify_data_table_permission(table_id, session=session)
engines = admin_logic.get_query_engines_by_environment(environment_id,
session=session)
engine_ids = [engine.id for engine in engines]
api_assert(engine_id is None or engine_id in engine_ids,
"Invalid engine id")
query_logs = logic.get_table_query_examples(
table_id,
engine_ids,
uid=uid,
engine_id=engine_id,
with_table_id=with_table_id,
limit=limit,
offset=offset,
session=session,
)
query_ids = [log.query_execution_id for log in query_logs]
return query_ids
def search_datadoc(
environment_id,
keywords,
filters=[],
fields=[],
sort_key=None,
sort_order=None,
limit=1000,
offset=0,
):
verify_environment_permission([environment_id])
filters.append(["environment_id", environment_id])
query = _construct_datadoc_query(
keywords=keywords,
filters=filters,
fields=fields,
limit=limit,
offset=offset,
sort_key=sort_key,
sort_order=sort_order,
)
results, count = _get_matching_objects(
query,
ES_CONFIG["datadocs"]["index_name"],
ES_CONFIG["datadocs"]["type_name"],
True,
)
return {"count": count, "results": results}
def search_query_snippet(environment_id, engine_id=None, is_public=False, golden=None):
search_by = current_user.id
with DBSession() as session:
# TODO: Check if engine_id is under environment_id
verify_environment_permission([environment_id])
engine_ids = (
[engine_id]
if engine_id is not None
else list(
map(
lambda e: e.id,
admin_logic.get_query_engines_by_environment(environment_id),
)
)
)
query_snippets = logic.search_snippet(
search_by=search_by,
engine_ids=engine_ids,
is_public=is_public,
golden=golden,
session=session,
)
query_snippet_dicts = [
dict(id=snippet[0], title=snippet[1]) for snippet in query_snippets
]
return query_snippet_dicts
def search_query(
environment_id,
keywords,
filters=[],
sort_key=None,
sort_order=None,
limit=1000,
offset=0,
):
verify_environment_permission([environment_id])
filters.append(["environment_id", environment_id])
query = _construct_query_search_query(
keywords=keywords,
filters=filters,
limit=limit,
offset=offset,
sort_key=sort_key,
sort_order=sort_order,
)
index_name = "{},{}".format(
ES_CONFIG["query_cells"]["index_name"],
ES_CONFIG["query_executions"]["index_name"],
)
results, count = _get_matching_objects(query, index_name, True)
return {"count": count, "results": results}
def create_table_samples(
table_id,
environment_id,
engine_id,
partition=None,
where=None,
order_by=None,
order_by_asc=True,
limit=100,
):
with DBSession() as session:
api_assert(limit <= 100, "Too many rows requested")
verify_environment_permission([environment_id])
verify_data_table_permission(table_id, session=session)
verify_query_engine_environment_permission(engine_id,
environment_id,
session=session)
task = run_sample_query.apply_async(args=[
table_id,
engine_id,
current_user.id,
limit,
partition,
where,
order_by,
order_by_asc,
])
return task.task_id
def get_board_by_id(board_id):
with DBSession() as session:
assert_can_read(board_id, session=session)
board = Board.get(id=board_id, session=session)
api_assert(board is not None, "Invalid board id", 404)
verify_environment_permission([board.environment_id])
return board.to_dict(extra_fields=["docs", "tables", "items"])
def create_data_doc_from_execution(
environment_id,
execution_id,
engine_id,
query_string,
title=None,
):
with DBSession() as session:
verify_environment_permission([environment_id])
environment = Environment.get(id=environment_id, session=session)
execution = get_query_execution_by_id(execution_id, session=session)
uid = current_user.id
api_assert(execution.uid == uid,
"You can only create from your own executions.")
return logic.create_data_doc_from_execution(
environment_id=environment_id,
owner_uid=uid,
engine_id=engine_id,
query_string=query_string,
execution_id=execution_id,
public=environment.shareable,
archived=False,
title=title,
meta={},
session=session,
)
def create_datadoc_schedule(
id,
cron,
kwargs,
):
kwargs_valid, kwargs_valid_reason = validate_datadoc_schedule_config(
kwargs)
api_assert(kwargs_valid, kwargs_valid_reason)
api_assert(validate_cron(cron), "Invalid cron expression")
schedule_name = schedule_logic.get_data_doc_schedule_name(id)
with DBSession() as session:
assert_can_write(id, session=session)
data_doc = logic.get_data_doc_by_id(id, session=session)
verify_environment_permission([data_doc.environment_id])
return schedule_logic.create_task_schedule(
schedule_name,
"tasks.run_datadoc.run_datadoc",
cron=cron,
kwargs={
**kwargs, "user_id": current_user.id,
"doc_id": id
},
task_type="user",
session=session,
)
def get_or_create_favorite_board(environment_id):
verify_environment_permission([environment_id])
with DBSession() as session:
board = logic.get_or_create_user_favorite_board(current_user.id,
environment_id,
session=session)
return board.to_dict(extra_fields=["docs", "tables"])
def search_board(
environment_id,
keywords,
filters=[],
fields=[],
sort_key=None,
sort_order=None,
limit=1000,
offset=0,
):
verify_environment_permission([environment_id])
filters.append(["environment_id", environment_id])
query = construct_board_query(
uid=current_user.id,
keywords=keywords,
filters=filters,
fields=fields,
limit=limit,
offset=offset,
sort_key=sort_key,
sort_order=sort_order,
)
results, count = get_matching_objects(query,
ES_CONFIG["boards"]["index_name"],
True)
return {"count": count, "results": results}
def get_data_docs(environment_id,
filter_mode=None,
offset=0,
limit=500,
archived=False):
with DBSession() as session:
verify_environment_permission([environment_id])
docs = []
if filter_mode == "mine":
docs = logic.get_data_doc_by_user(
current_user.id,
environment_id=environment_id,
offset=offset,
limit=limit,
session=session,
)
elif filter_mode == "favorite":
docs = logic.get_user_favorite_data_docs(
current_user.id,
environment_id=environment_id,
session=session)
elif filter_mode == "recent":
docs = logic.get_user_recent_data_docs(
current_user.id,
environment_id=environment_id,
session=session)
return docs
def search_datadoc(
environment_id,
keywords,
filters=[],
fields=[],
sort_key=None,
sort_order=None,
limit=1000,
offset=0,
):
verify_environment_permission([environment_id])
filters.append(["environment_id", environment_id])
# Unfortuantely currently we can't search including underscore,
# so split. # TODO: Allow for both.
# parsed_keywords = map(lambda x: re.split('(-|_)', x), keywords)
query = _construct_datadoc_query(
keywords=keywords,
filters=filters,
fields=fields,
limit=limit,
offset=offset,
sort_key=sort_key,
sort_order=sort_order,
)
results, count = _get_matching_objects(
query,
ES_CONFIG["datadocs"]["index_name"],
ES_CONFIG["datadocs"]["type_name"],
True,
)
return {"count": count, "results": results}
def search_query_execution(environment_id,
filters={},
orderBy=None,
limit=100,
offset=0):
verify_environment_permission([environment_id])
with DBSession() as session:
if "user" in filters:
api_assert(
current_user.id == filters["user"],
"You can only search your own queries",
)
else:
filters["user"] = current_user.id
query_executions = logic.search_query_execution(
environment_id=environment_id,
filters=filters,
orderBy=orderBy,
limit=limit,
offset=offset,
session=session,
)
result = [
query_execution.to_dict(with_statement=False)
for query_execution in query_executions
]
return result
def soft_delete_data_doc(id):
with DBSession() as session:
doc = logic.get_data_doc_by_id(id=id, session=session)
api_assert(doc, "Invalid doc")
verify_environment_permission([doc.environment_id])
api_assert(current_user.id == doc.owner_uid,
"You can only delete your own data doc")
logic.update_data_doc(id=id, archived=True, session=session)
def on_join_room(data_doc_id: str):
data_doc_id = int(data_doc_id)
with DBSession() as session:
assert_can_read(data_doc_id, session=session)
data_doc = logic.get_data_doc_by_id(data_doc_id, session=session)
verify_environment_permission([data_doc.environment_id])
join_room(data_doc_id)
update_user_list(data_doc_id, add=True)
def get_table_query_examples_users(table_id, environment_id, limit=5):
api_assert(limit <= 10)
verify_environment_permission([environment_id])
verify_data_table_permission(table_id)
engines = admin_logic.get_query_engines_by_environment(environment_id)
engine_ids = [engine.id for engine in engines]
users = logic.get_query_example_users(table_id, engine_ids, limit=limit)
return [{"uid": r[0], "count": r[1]} for r in users]
def get_table_samples(table_id, environment_id):
try:
with DBSession() as session:
verify_environment_permission([environment_id])
verify_data_table_permission(table_id, session=session)
return mysql_cache.get_key(
f"table_samples_{table_id}_{current_user.id}",
expires_after=seconds_in_a_day,
session=session,
)
except LookupError:
return None
def get_boards_from_board_item(item_type: str, item_id: int,
environment_id: int):
"""Given a potential item, find all boards containing the item
that the current user has access to
Arguments:
item_type {[str]} -- [data_doc or table]
item_id {[int]} -- [Doc id or table id]
environment_id {[int]} - [id of board environment]
"""
verify_environment_permission([environment_id])
return logic.get_boards_from_board_item(item_type, item_id, environment_id,
current_user.id)
def create_data_doc(environment_id, cells=[], title=None):
with DBSession() as session:
verify_environment_permission([environment_id])
environment = Environment.get(id=environment_id, session=session)
return logic.create_data_doc(
environment_id=environment_id,
owner_uid=current_user.id,
cells=cells,
public=environment.shareable,
archived=False,
title=title,
meta={},
session=session,
)
def get_my_datadoc_with_schedule(environment_id,
offset=0,
limit=10,
filters=None):
assert limit <= 100, "Too many docs"
verify_environment_permission([environment_id])
docs = []
docs, count = schedule_logic.get_scheduled_data_docs_by_user(
current_user.id,
environment_id=environment_id,
offset=offset,
limit=limit,
filters=filters,
)
return {"docs": docs, "count": count}
def get_board_by_id(board_id, environment_id):
with DBSession() as session:
if board_id == 0:
verify_environment_permission([environment_id])
public_boards = logic.get_all_public_boards(
environment_id=environment_id, session=session)
return {
"id": 0,
"boards": [public_board.id for public_board in public_boards],
}
assert_can_read(board_id, session=session)
board = Board.get(id=board_id, session=session)
api_assert(board is not None, "Invalid board id", 404)
verify_environment_permission([board.environment_id])
return board.to_dict(
extra_fields=["docs", "tables", "boards", "queries", "items"])
def create_board(
name,
environment_id,
description=None,
public=None,
favorite=False,
):
with DBSession() as session:
verify_environment_permission([environment_id])
return logic.create_board(
name,
environment_id,
current_user.id,
description,
public,
favorite,
session=session,
).to_dict()
def update_data_cell(cell_id, fields, sid="", session=None):
data_doc = logic.get_data_doc_by_data_cell_id(cell_id, session=session)
assert_can_write(data_doc.id, session=session)
verify_environment_permission([data_doc.environment_id])
data_cell = logic.update_data_cell(
id=cell_id,
session=session,
**fields,
)
data_cell_dict = data_cell.to_dict()
socketio.emit(
"data_cell_updated",
(sid, data_cell_dict),
namespace=DATA_DOC_NAMESPACE,
room=data_doc.id,
broadcast=True,
)
return data_cell_dict
def get_datadoc(doc_id, session=None):
assert_can_read(doc_id, session=session)
doc = logic.get_data_doc_by_id(id=doc_id, session=session)
if doc:
verify_environment_permission([doc.environment_id])
return doc.to_dict(with_cells=True)
def paste_data_cell(cell_id: int,
cut: bool,
doc_id: int,
index: int,
sid="",
session=None):
data_cell = logic.get_data_cell_by_id(cell_id, session=session)
assert data_cell is not None, "Data cell does not exist"
data_doc = logic.get_data_doc_by_id(doc_id, session=session)
old_data_doc = data_cell.doc
same_doc = old_data_doc.id == doc_id
# Make sure they are in the same environment and have access
assert (old_data_doc.environment_id == data_doc.environment_id
), "Must be in the same environment"
verify_environment_permission([data_doc.environment_id])
# Users need to be able to write in the doc copied to
assert_can_write(doc_id, session=session)
if not same_doc:
if cut:
# To cut the user need to be able to write the original doc
assert_can_write(old_data_doc.id, session=session)
else:
# To copy the user need to be able to read the original doc
assert_can_read(old_data_doc.id, session=session)
if cut:
old_cell_index = logic.get_data_doc_data_cell(
cell_id, session=session).cell_order
logic.move_data_doc_cell_to_doc(cell_id,
doc_id,
index,
session=session)
if same_doc:
# Account for shift in original index
# See more details in move_data_doc_cell_to_doc
if old_cell_index < index:
index -= 1
socketio.emit(
"data_cell_moved",
# sid, from_index, to_index
(
sid,
old_cell_index,
index,
),
namespace=DATA_DOC_NAMESPACE,
room=doc_id,
broadcast=True,
)
else:
socketio.emit(
"data_cell_inserted",
(
sid,
index,
data_cell.to_dict(),
),
namespace=DATA_DOC_NAMESPACE,
room=doc_id,
broadcast=True,
)
socketio.emit(
"data_cell_deleted",
(
sid,
cell_id,
),
namespace=DATA_DOC_NAMESPACE,
room=old_data_doc.id,
broadcast=True,
)
else: # Copy
new_cell_dict = insert_data_cell(
doc_id,
index,
data_cell.cell_type.name,
data_cell.context,
data_cell.meta,
sid,
session=session,
)
# Copy all query history over
logic.copy_cell_history(cell_id, new_cell_dict["id"], session=session)
# To resolve the sender's promise
socketio.emit(
"data_cell_pasted",
(sid),
namespace=DATA_DOC_NAMESPACE,
room=doc_id,
broadcast=False,
)
def get_query_engines(environment_id):
verify_environment_permission([environment_id])
return admin_logic.get_query_engines_by_environment(environment_id,
ordered=True)
def get_all_query_metastores(environment_id, ):
verify_environment_permission([environment_id])
return admin_logic.get_all_query_metastore_by_environment(environment_id)
