def get_table_query_examples_engines(table_id, environment_id): verify_environment_permission([environment_id]) verify_data_table_permission(table_id) engines = logic.get_query_example_engines(table_id, environment_id=environment_id) return [{"engine_id": r[0], "count": r[1]} for r in engines]
def get_table_query_examples( table_id, environment_id, uid=None, engine_id=None, with_table_id=None, limit=10, offset=0, ): api_assert(limit < 100) with DBSession() as session: verify_environment_permission([environment_id]) verify_data_table_permission(table_id, session=session) engines = admin_logic.get_query_engines_by_environment(environment_id, session=session) engine_ids = [engine.id for engine in engines] api_assert(engine_id is None or engine_id in engine_ids, "Invalid engine id") query_logs = logic.get_table_query_examples( table_id, engine_ids, uid=uid, engine_id=engine_id, with_table_id=with_table_id, limit=limit, offset=offset, session=session, ) query_ids = [log.query_execution_id for log in query_logs] return query_ids
def search_datadoc( environment_id, keywords, filters=[], fields=[], sort_key=None, sort_order=None, limit=1000, offset=0, ): verify_environment_permission([environment_id]) filters.append(["environment_id", environment_id]) query = _construct_datadoc_query( keywords=keywords, filters=filters, fields=fields, limit=limit, offset=offset, sort_key=sort_key, sort_order=sort_order, ) results, count = _get_matching_objects( query, ES_CONFIG["datadocs"]["index_name"], ES_CONFIG["datadocs"]["type_name"], True, ) return {"count": count, "results": results}
def search_query_snippet(environment_id, engine_id=None, is_public=False, golden=None): search_by = current_user.id with DBSession() as session: # TODO: Check if engine_id is under environment_id verify_environment_permission([environment_id]) engine_ids = ( [engine_id] if engine_id is not None else list( map( lambda e: e.id, admin_logic.get_query_engines_by_environment(environment_id), ) ) ) query_snippets = logic.search_snippet( search_by=search_by, engine_ids=engine_ids, is_public=is_public, golden=golden, session=session, ) query_snippet_dicts = [ dict(id=snippet[0], title=snippet[1]) for snippet in query_snippets ] return query_snippet_dicts
def search_query( environment_id, keywords, filters=[], sort_key=None, sort_order=None, limit=1000, offset=0, ): verify_environment_permission([environment_id]) filters.append(["environment_id", environment_id]) query = _construct_query_search_query( keywords=keywords, filters=filters, limit=limit, offset=offset, sort_key=sort_key, sort_order=sort_order, ) index_name = "{},{}".format( ES_CONFIG["query_cells"]["index_name"], ES_CONFIG["query_executions"]["index_name"], ) results, count = _get_matching_objects(query, index_name, True) return {"count": count, "results": results}
def create_table_samples( table_id, environment_id, engine_id, partition=None, where=None, order_by=None, order_by_asc=True, limit=100, ): with DBSession() as session: api_assert(limit <= 100, "Too many rows requested") verify_environment_permission([environment_id]) verify_data_table_permission(table_id, session=session) verify_query_engine_environment_permission(engine_id, environment_id, session=session) task = run_sample_query.apply_async(args=[ table_id, engine_id, current_user.id, limit, partition, where, order_by, order_by_asc, ]) return task.task_id
def get_board_by_id(board_id): with DBSession() as session: assert_can_read(board_id, session=session) board = Board.get(id=board_id, session=session) api_assert(board is not None, "Invalid board id", 404) verify_environment_permission([board.environment_id]) return board.to_dict(extra_fields=["docs", "tables", "items"])
def create_data_doc_from_execution( environment_id, execution_id, engine_id, query_string, title=None, ): with DBSession() as session: verify_environment_permission([environment_id]) environment = Environment.get(id=environment_id, session=session) execution = get_query_execution_by_id(execution_id, session=session) uid = current_user.id api_assert(execution.uid == uid, "You can only create from your own executions.") return logic.create_data_doc_from_execution( environment_id=environment_id, owner_uid=uid, engine_id=engine_id, query_string=query_string, execution_id=execution_id, public=environment.shareable, archived=False, title=title, meta={}, session=session, )
def create_datadoc_schedule( id, cron, kwargs, ): kwargs_valid, kwargs_valid_reason = validate_datadoc_schedule_config( kwargs) api_assert(kwargs_valid, kwargs_valid_reason) api_assert(validate_cron(cron), "Invalid cron expression") schedule_name = schedule_logic.get_data_doc_schedule_name(id) with DBSession() as session: assert_can_write(id, session=session) data_doc = logic.get_data_doc_by_id(id, session=session) verify_environment_permission([data_doc.environment_id]) return schedule_logic.create_task_schedule( schedule_name, "tasks.run_datadoc.run_datadoc", cron=cron, kwargs={ **kwargs, "user_id": current_user.id, "doc_id": id }, task_type="user", session=session, )
def get_or_create_favorite_board(environment_id): verify_environment_permission([environment_id]) with DBSession() as session: board = logic.get_or_create_user_favorite_board(current_user.id, environment_id, session=session) return board.to_dict(extra_fields=["docs", "tables"])
def search_board( environment_id, keywords, filters=[], fields=[], sort_key=None, sort_order=None, limit=1000, offset=0, ): verify_environment_permission([environment_id]) filters.append(["environment_id", environment_id]) query = construct_board_query( uid=current_user.id, keywords=keywords, filters=filters, fields=fields, limit=limit, offset=offset, sort_key=sort_key, sort_order=sort_order, ) results, count = get_matching_objects(query, ES_CONFIG["boards"]["index_name"], True) return {"count": count, "results": results}
def get_data_docs(environment_id, filter_mode=None, offset=0, limit=500, archived=False): with DBSession() as session: verify_environment_permission([environment_id]) docs = [] if filter_mode == "mine": docs = logic.get_data_doc_by_user( current_user.id, environment_id=environment_id, offset=offset, limit=limit, session=session, ) elif filter_mode == "favorite": docs = logic.get_user_favorite_data_docs( current_user.id, environment_id=environment_id, session=session) elif filter_mode == "recent": docs = logic.get_user_recent_data_docs( current_user.id, environment_id=environment_id, session=session) return docs
def search_datadoc( environment_id, keywords, filters=[], fields=[], sort_key=None, sort_order=None, limit=1000, offset=0, ): verify_environment_permission([environment_id]) filters.append(["environment_id", environment_id]) # Unfortuantely currently we can't search including underscore, # so split. # TODO: Allow for both. # parsed_keywords = map(lambda x: re.split('(-|_)', x), keywords) query = _construct_datadoc_query( keywords=keywords, filters=filters, fields=fields, limit=limit, offset=offset, sort_key=sort_key, sort_order=sort_order, ) results, count = _get_matching_objects( query, ES_CONFIG["datadocs"]["index_name"], ES_CONFIG["datadocs"]["type_name"], True, ) return {"count": count, "results": results}
def search_query_execution(environment_id, filters={}, orderBy=None, limit=100, offset=0): verify_environment_permission([environment_id]) with DBSession() as session: if "user" in filters: api_assert( current_user.id == filters["user"], "You can only search your own queries", ) else: filters["user"] = current_user.id query_executions = logic.search_query_execution( environment_id=environment_id, filters=filters, orderBy=orderBy, limit=limit, offset=offset, session=session, ) result = [ query_execution.to_dict(with_statement=False) for query_execution in query_executions ] return result
def soft_delete_data_doc(id): with DBSession() as session: doc = logic.get_data_doc_by_id(id=id, session=session) api_assert(doc, "Invalid doc") verify_environment_permission([doc.environment_id]) api_assert(current_user.id == doc.owner_uid, "You can only delete your own data doc") logic.update_data_doc(id=id, archived=True, session=session)
def on_join_room(data_doc_id: str): data_doc_id = int(data_doc_id) with DBSession() as session: assert_can_read(data_doc_id, session=session) data_doc = logic.get_data_doc_by_id(data_doc_id, session=session) verify_environment_permission([data_doc.environment_id]) join_room(data_doc_id) update_user_list(data_doc_id, add=True)
def get_table_query_examples_users(table_id, environment_id, limit=5): api_assert(limit <= 10) verify_environment_permission([environment_id]) verify_data_table_permission(table_id) engines = admin_logic.get_query_engines_by_environment(environment_id) engine_ids = [engine.id for engine in engines] users = logic.get_query_example_users(table_id, engine_ids, limit=limit) return [{"uid": r[0], "count": r[1]} for r in users]
def get_table_samples(table_id, environment_id): try: with DBSession() as session: verify_environment_permission([environment_id]) verify_data_table_permission(table_id, session=session) return mysql_cache.get_key( f"table_samples_{table_id}_{current_user.id}", expires_after=seconds_in_a_day, session=session, ) except LookupError: return None
def get_boards_from_board_item(item_type: str, item_id: int, environment_id: int): """Given a potential item, find all boards containing the item that the current user has access to Arguments: item_type {[str]} -- [data_doc or table] item_id {[int]} -- [Doc id or table id] environment_id {[int]} - [id of board environment] """ verify_environment_permission([environment_id]) return logic.get_boards_from_board_item(item_type, item_id, environment_id, current_user.id)
def create_data_doc(environment_id, cells=[], title=None): with DBSession() as session: verify_environment_permission([environment_id]) environment = Environment.get(id=environment_id, session=session) return logic.create_data_doc( environment_id=environment_id, owner_uid=current_user.id, cells=cells, public=environment.shareable, archived=False, title=title, meta={}, session=session, )
def get_my_datadoc_with_schedule(environment_id, offset=0, limit=10, filters=None): assert limit <= 100, "Too many docs" verify_environment_permission([environment_id]) docs = [] docs, count = schedule_logic.get_scheduled_data_docs_by_user( current_user.id, environment_id=environment_id, offset=offset, limit=limit, filters=filters, ) return {"docs": docs, "count": count}
def get_board_by_id(board_id, environment_id): with DBSession() as session: if board_id == 0: verify_environment_permission([environment_id]) public_boards = logic.get_all_public_boards( environment_id=environment_id, session=session) return { "id": 0, "boards": [public_board.id for public_board in public_boards], } assert_can_read(board_id, session=session) board = Board.get(id=board_id, session=session) api_assert(board is not None, "Invalid board id", 404) verify_environment_permission([board.environment_id]) return board.to_dict( extra_fields=["docs", "tables", "boards", "queries", "items"])
def create_board( name, environment_id, description=None, public=None, favorite=False, ): with DBSession() as session: verify_environment_permission([environment_id]) return logic.create_board( name, environment_id, current_user.id, description, public, favorite, session=session, ).to_dict()
def update_data_cell(cell_id, fields, sid="", session=None): data_doc = logic.get_data_doc_by_data_cell_id(cell_id, session=session) assert_can_write(data_doc.id, session=session) verify_environment_permission([data_doc.environment_id]) data_cell = logic.update_data_cell( id=cell_id, session=session, **fields, ) data_cell_dict = data_cell.to_dict() socketio.emit( "data_cell_updated", (sid, data_cell_dict), namespace=DATA_DOC_NAMESPACE, room=data_doc.id, broadcast=True, ) return data_cell_dict
def get_datadoc(doc_id, session=None): assert_can_read(doc_id, session=session) doc = logic.get_data_doc_by_id(id=doc_id, session=session) if doc: verify_environment_permission([doc.environment_id]) return doc.to_dict(with_cells=True)
def paste_data_cell(cell_id: int, cut: bool, doc_id: int, index: int, sid="", session=None): data_cell = logic.get_data_cell_by_id(cell_id, session=session) assert data_cell is not None, "Data cell does not exist" data_doc = logic.get_data_doc_by_id(doc_id, session=session) old_data_doc = data_cell.doc same_doc = old_data_doc.id == doc_id # Make sure they are in the same environment and have access assert (old_data_doc.environment_id == data_doc.environment_id ), "Must be in the same environment" verify_environment_permission([data_doc.environment_id]) # Users need to be able to write in the doc copied to assert_can_write(doc_id, session=session) if not same_doc: if cut: # To cut the user need to be able to write the original doc assert_can_write(old_data_doc.id, session=session) else: # To copy the user need to be able to read the original doc assert_can_read(old_data_doc.id, session=session) if cut: old_cell_index = logic.get_data_doc_data_cell( cell_id, session=session).cell_order logic.move_data_doc_cell_to_doc(cell_id, doc_id, index, session=session) if same_doc: # Account for shift in original index # See more details in move_data_doc_cell_to_doc if old_cell_index < index: index -= 1 socketio.emit( "data_cell_moved", # sid, from_index, to_index ( sid, old_cell_index, index, ), namespace=DATA_DOC_NAMESPACE, room=doc_id, broadcast=True, ) else: socketio.emit( "data_cell_inserted", ( sid, index, data_cell.to_dict(), ), namespace=DATA_DOC_NAMESPACE, room=doc_id, broadcast=True, ) socketio.emit( "data_cell_deleted", ( sid, cell_id, ), namespace=DATA_DOC_NAMESPACE, room=old_data_doc.id, broadcast=True, ) else: # Copy new_cell_dict = insert_data_cell( doc_id, index, data_cell.cell_type.name, data_cell.context, data_cell.meta, sid, session=session, ) # Copy all query history over logic.copy_cell_history(cell_id, new_cell_dict["id"], session=session) # To resolve the sender's promise socketio.emit( "data_cell_pasted", (sid), namespace=DATA_DOC_NAMESPACE, room=doc_id, broadcast=False, )
def get_query_engines(environment_id): verify_environment_permission([environment_id]) return admin_logic.get_query_engines_by_environment(environment_id, ordered=True)
def get_all_query_metastores(environment_id, ): verify_environment_permission([environment_id]) return admin_logic.get_all_query_metastore_by_environment(environment_id)