def onboard_file_bundle(kiara: Kiara, uploaded_files, aliases: typing.Optional[typing.Iterable[str]]=None):
if not uploaded_files:
return
if aliases is None:
aliases = []
if isinstance(aliases, str):
aliases = [aliases]
bundle_aliases = [f"{x}_uploaded_files" for x in aliases]
if isinstance(uploaded_files, UploadedFile):
uploaded_files = [uploaded_files]
with tempfile.TemporaryDirectory() as tmpdirname:
for uf in uploaded_files:
path = os.path.join(tmpdirname, uf.name)
with open(path, 'wb') as f:
f.write(uf.getbuffer())
inputs = {
"path": tmpdirname,
"aliases": aliases,
"file_bundle_aliases": bundle_aliases
}
kiara.run("table.import.from_local_folder", inputs=inputs)
shutil.rmtree(tmpdirname, ignore_errors=True)
def init_session(
st,
module_type: str,
module_config: typing.Optional[typing.Mapping[str, typing.Any]] = None,
):
if "kiara" not in st.session_state:
print("Create kiara object.")
kiara = Kiara()
st.session_state["kiara"] = kiara
else:
kiara = st.session_state["kiara"]
if "workflow" not in st.session_state:
print(f"Create workflow in session: {module_type}")
controller = BatchControllerManual()
workflow: KiaraWorkflow = kiara.create_workflow(
module_type, module_config=module_config, controller=controller
)
st.session_state["workflow"] = workflow
else:
workflow = st.session_state["workflow"]
return (
kiara,
workflow,
)
def run_sql(kiara: Kiara, table_value: Value, unit: str):
pipeline_file = os.path.join(
os.path.dirname(__file__), "..", "..", "..", "newspaper_corpora", f"query_newspaper_corpora_{unit}.json"
)
workflow = kiara.create_workflow(pipeline_file)
workflow.inputs.set_value("value_id", table_value.id)
return workflow.outputs.get_value_obj("query_result")
def augment_table(kiara: Kiara, table_value: Value):
pipeline_file = os.path.join(
os.path.dirname(__file__), "..", "..", "..", "newspaper_corpora", "augment_newspaper_corpora_table.json"
)
workflow = kiara.create_workflow(pipeline_file)
workflow.inputs.set_value("value_id", table_value.id)
return workflow.outputs.get_value_obj("table")
def import_bytes(kiara: Kiara, uploaded_file: UploadedFile):
with tempfile.TemporaryDirectory() as tmpdirname:
path = os.path.join(tmpdirname, uploaded_file.name)
with open(path, "wb") as f:
f.write(uploaded_file.getbuffer())
if uploaded_file.name.endswith(".csv"):
kiara.run(
"table.import.from_local_file",
inputs={
"path": path,
"aliases": [uploaded_file.name[0:-4]],
"file_aliases": [uploaded_file.name],
},
)
else:
kiara.run(
"import.local_file",
inputs={"path": path, "aliases": [uploaded_file.name]},
)
def __init__(self, streamlit):
self._streamlit = streamlit
if "kiara" not in self._streamlit.session_state:
print("Create kiara object.")
self._kiara = Kiara()
self._streamlit.session_state["kiara"] = self._kiara
else:
self._kiara = self._streamlit.session_state["kiara"]
self._pages = {}
# -*- coding: utf-8 -*-
# run with:
#
# streamlit run examples/streamlit/streamlit.py
import streamlit as st
from kiara import Kiara
from kiara.workflow.kiara_workflow import KiaraWorkflow
from streamlit.uploaded_file_manager import UploadedFile
st.title("Kiara/streamlit experiment - Network analysis")
if "kiara" not in st.session_state:
print("CREATE KIARA")
kiara = Kiara()
st.session_state["kiara"] = kiara
else:
kiara = st.session_state["kiara"]
if "workflow" not in st.session_state:
print("CREATE WORKFLOW")
workflow: KiaraWorkflow = kiara.create_workflow(
"/home/markus/projects/dharpa/kiara_modules.playground/examples/pipelines/network_analysis_end_to_end.json"
)
st.session_state["workflow"] = workflow
else:
workflow = st.session_state["workflow"]
def add_file(f):
import pandas as pd
import pyarrow as pa
import streamlit as st
from kiara import Kiara
# before running this, the dataset must be imported into the local kiara data store, with the id_or_alias 'df_distrib':
#
# kiara run table.import.from_local_file path=/home/markus/projects/dharpa/kiara_modules.playground/data/mariella/v2/df_distrib-3.csv ids_or_aliases=df_distrib
from kiara.data import Value
from pandas import DataFrame
from streamlit_observable import observable
# st.markdown('<style>@import url("https://fonts.googleapis.com/css2?family=Palanquin:wght@400;700&display=swap")</style>', unsafe_allow_html=True)
# selection = st.sidebar.radio("Go to", list(PAGES.keys()))
kiara = Kiara.instance()
table_value: Value = kiara.data_store.load_value("df_distrib")
table: pa.Table = table_value.get_value_data()
df: DataFrame = table.to_pandas()
data = list(df.to_dict(orient="index").values())
data_json = json.dumps(data, default=str)
cleaned_data = json.loads(data_json)
timeSelected = st.sidebar.radio(label="Select time span",
index=3,
options=["day", "week", "month", "year"])
observers = observable(
def create_graph(kiara: Kiara):
def create_graph(alias, edges, nodes, source_column, target_column,
weight_column, node_index):
if not alias:
return ("No alias specified, doing nothing...", None)
all_graph_aliases = find_all_aliases_of_type(
kiara, value_type="network_graph")
if alias in all_graph_aliases:
return (f"Alias '{alias}' already registered.", None)
if not edges:
return ("No edges table specified, doing nothing...", None)
inputs = {
"edges_table_id": edges,
"nodes_table_id": nodes,
"source_column": source_column,
"target_column": target_column,
"weight_column": weight_column,
"index_column_name": node_index,
}
try:
workflow = kiara.create_workflow(pipeline_file)
workflow.inputs.set_values(**inputs)
graph_value = workflow.outputs.get_value_obj("graph")
graph_value.save([alias])
return (f"", graph_value)
except Exception as e:
return (f"Error creating graph: {e}", None)
return ("CREATED GRAPH", None)
def get_table_column_names(table_id):
if not table_id:
return []
md = kiara.data_store.get_metadata_for_id(table_id)
if not md:
return []
return md.metadata["table"]["metadata_item"]["column_names"]
def find_likely_index(options: typing.Iterable, keyword: str):
for idx, alias in enumerate(options):
if keyword.lower() in alias.lower():
return idx
return 0
graph = None
st.write("Create a new graph")
graph_alias = st.text_input("The alias for the graph")
all_table_aliases = find_all_aliases_of_type(kiara, value_type="table")
default_edge_table = find_likely_index(all_table_aliases, "edge")
default_node_table = find_likely_index(all_table_aliases, "node")
select_edges = st.selectbox("Edges",
all_table_aliases,
index=default_edge_table)
select_nodes = st.selectbox("Nodes",
all_table_aliases,
index=default_node_table)
edge_column_names = get_table_column_names(select_edges)
nodes_column_names = get_table_column_names(select_nodes)
default_source_name = find_likely_index(edge_column_names, "source")
default_target_name = find_likely_index(edge_column_names, "target")
default_weight_name = find_likely_index(edge_column_names, "weight")
default_id_name = find_likely_index(nodes_column_names, "id")
source_column_name = st.selectbox("Source column name",
edge_column_names,
index=default_source_name)
target_column_name = st.selectbox("Target column name",
edge_column_names,
index=default_target_name)
weight_column_name = st.selectbox("Weight column name",
edge_column_names,
index=default_weight_name)
nodes_index_name = st.selectbox("Nodes table_index",
nodes_column_names,
index=default_id_name)
create_button = st.button(label="Create graph")
if create_button:
result, graph = create_graph(
alias=graph_alias,
edges=select_edges,
nodes=select_nodes,
source_column=source_column_name,
target_column=target_column_name,
weight_column=weight_column_name,
node_index=nodes_index_name,
)
st.write(result)
if graph is None:
return
st.write("Graph properties")
props = kiara.run("network.graph.properties",
inputs={"graph": graph},
resolve_result=True)
st.write(props)
operation_id: str,
container: DeltaGenerator = st):
# this retrieve the object for the operation we are interested in
operation: Operation = kiara.operation_mgmt.profiles[operation_id]
# this is all operation specific data that is available, pick what you need
container.markdown("## Operation info")
container.markdown("### Type metadata")
type_metadata = operation.module_cls.get_type_metadata()
container.write(type_metadata.dict())
# you can access specific elements via the Python object attributes, like:
# type_metadata.context.references
container.markdown(("### Inputs"))
for field_name, schema in operation.input_schemas.items():
container.write(f"#### {field_name}")
container.write(schema.dict())
container.markdown(("### Outputs"))
for field_name, schema in operation.output_schemas.items():
container.write(f"#### {field_name}")
container.write(schema.dict())
kiara = Kiara()
# get all module types
kiara_module_types = kiara.operation_mgmt.profiles.keys()
selected_operation = st.selectbox("Select the module type", kiara_module_types)
write_operation_info(kiara=kiara, operation_id=selected_operation)
# )
# data = observers.get("corpus_agg")
# df = pd.DataFrame(data)
# st.dataframe(df)
#
#
# uploaded_files = st.file_uploader("Add files", type = 'txt', accept_multiple_files=True)
# print(uploaded_files)
from kiara import Kiara
path = st.text_input("Path")
button = st.button("Import")
if button:
kiara = Kiara()
module = kiara.create_module("table.import.from_local_folder")
result = module.run(path=path)
table_obj = result.get_value_obj("table")
aliases = ['my_first_table']
saved_metadata = table_obj.save(aliases=aliases)
st.write(saved_metadata.dict())
# table = result.get_value_data("table")
# st.write(table.to_pandas())
# if not st.session_state.get("counter"):
# st.session_state.counter = 0
#
# something_else = st.checkbox("Just an example")
#