def test_complex_df_report():
"""Test our dataframe importing with types of DFs user's upload"""
tz_df = pd.DataFrame(
dict(
duration_col=[timedelta(seconds=x) for x in range(30)],
date_col=[date.today() for _ in range(30)],
datetime_col=[datetime.utcnow() for _ in range(30)],
datetimez_col=[datetime.now(timezone.utc) for _ in range(30)],
)
)
raw_data = {
"first_name": ["Jason", "Molly", "Tina", "Jake", "Amy"],
"last_name": ["Miller", "Jacobson", "Ali", "Milner", "Cooze"],
"age": [42, 52, 36, 24, 73],
"preTestScore": [4, 24, 31, 2, 3],
"postTestScore": [25, 94, 57, 62, 70],
}
index_df = pd.DataFrame(raw_data, columns=["first_name", "last_name", "age", "preTestScore", "postTestScore"])
df_desc = index_df.describe()
df_desc_2 = df_desc.reset_index()
tz_t = dp.DataTable(tz_df)
index_t = dp.DataTable(index_df)
df_desc_t = dp.DataTable(df_desc)
df_desc_2_t = dp.DataTable(df_desc_2)
with deletable(dp.Report(tz_t, index_t, df_desc_t, df_desc_2_t)) as dp_report:
dp_report.upload(name=gen_name())
def test_gen_failing_reports():
# local reports with unsupported elements
with pytest.raises(DPError):
r = dp.Report(dp.DataTable(gen_df()))
r._gen_report(embedded=True, title="TITLE", description="DESCRIPTION")
# nested pages
with pytest.raises((DocumentInvalid, DPError)):
r = dp.Report(dp.Page(dp.Page(md_block)))
r._gen_report(embedded=False, title="TITLE", description="DESCRIPTION")
with pytest.raises((DocumentInvalid, DPError)):
r = dp.Report(dp.Group(dp.Page(md_block)))
r._gen_report(embedded=False, title="TITLE", description="DESCRIPTION")
# group with 0 object
with pytest.raises((DocumentInvalid, DPError)):
r = dp.Report(dp.Page(dp.Group(blocks=[])))
r._gen_report(embedded=False, title="TITLE", description="DESCRIPTION")
# select with 1 object
with pytest.raises((DocumentInvalid, DPError)):
r = dp.Report(dp.Page(dp.Select(blocks=[md_block])))
r._gen_report(embedded=False, title="TITLE", description="DESCRIPTION")
# empty text block
with pytest.raises((AssertionError, DocumentInvalid, DPError)):
r = dp.Report(dp.Text(" "))
r._gen_report(embedded=False, title="TITLE", description="DESCRIPTION")
# empty df
with pytest.raises((AssertionError, DocumentInvalid, DPError)):
r = dp.Report(dp.DataTable(pd.DataFrame()))
r._gen_report(embedded=False, title="TITLE", description="DESCRIPTION")
def test_complex_df_report():
"""Test our dataframe importing with types of DFs user's upload"""
tz_df = convert_csv_pd(
"""
date,datetime,datetime_tz
2017-01-10,2017-01-21T23:10:24,2020-03-23T00:00:00.000Z
2017-01-11,2017-01-23T23:01:24,2020-04-23T00:00:00.000Z
"""
)
raw_data = {
"first_name": ["Jason", "Molly", "Tina", "Jake", "Amy"],
"last_name": ["Miller", "Jacobson", "Ali", "Milner", "Cooze"],
"age": [42, 52, 36, 24, 73],
"preTestScore": [4, 24, 31, 2, 3],
"postTestScore": [25, 94, 57, 62, 70],
}
index_df = pd.DataFrame(raw_data, columns=["first_name", "last_name", "age", "preTestScore", "postTestScore"])
df_desc = index_df.describe()
df_desc_2 = df_desc.reset_index()
tz_t = dp.DataTable(tz_df)
index_t = dp.DataTable(index_df)
df_desc_t = dp.DataTable(df_desc)
df_desc_2_t = dp.DataTable(df_desc_2)
with deletable(dp.Report(tz_t, index_t, df_desc_t, df_desc_2_t)) as dp_report:
dp_report.publish(name=gen_name())
def test_full_report(tmp_path: Path):
df = gen_df()
name = gen_name()
description = gen_description()
source_url = "https://github.com/datapane/datapane"
# create a basic report
m = dp.Text("hello world!!")
# Asset tests
lis = [1, 2, 3]
json_list: str = json.dumps(lis)
plot = gen_plot()
# create the DP
fn = tmp_path / "json_list.json"
fn.write_text(data=json_list)
file_asset = dp.File(file=fn)
json_asset = dp.File(data=json_list, is_json=True)
plot_asset = dp.Plot(data=plot)
list_asset = dp.File(data=lis, is_json=True)
df_asset = dp.DataTable(df=df, caption="Our Dataframe")
dp_report = dp.Report(m, file_asset, df_asset, json_asset, plot_asset, list_asset)
dp_report.upload(name=name, description=description, source_url=source_url)
with deletable(dp_report):
# are the fields ok
check_name(dp_report, name)
assert dp_report.description == description
assert dp_report.source_url == source_url
assert len(dp_report.pages[0].blocks[0].blocks) == 6
def gen_report_with_files(datadir: Path,
single_file: bool = False) -> dp.Report:
# Asset tests
lis = [1, 2, 3]
small_df = gen_df()
big_df = gen_df(10000)
# text
md_block = dp.Markdown(
text="# Test markdown block </hello> \n Test **content**")
html_block = dp.HTML(html="Hello World</hello>")
big_number = dp.BigNumber(heading="Tests written", value=1234)
big_number_1 = dp.BigNumber(heading="Real Tests written :)",
value=11,
change=2,
is_upward_change=True)
# assets
plot_asset = dp.Plot(data=alt.Chart(gen_df()).mark_line().encode(x="x",
y="y"),
caption="Plot Asset")
list_asset = dp.File(data=lis, name="List Asset", is_json=True)
img_asset = dp.File(file=datadir / "datapane-logo.png")
# tables
table_asset = dp.Table(data=small_df, caption="Test Basic Table")
dt_asset = dp.DataTable(df=big_df, caption="Test DataTable")
dt_pivot_asset = dp.DataTable(df=big_df,
caption="Test DataTable with Pivot",
can_pivot=True)
if single_file:
return dp.Report(dp.Blocks(blocks=[md_block, plot_asset]))
else:
return dp.Report(
md_block,
html_block,
big_number,
big_number_1,
plot_asset,
list_asset,
img_asset,
table_asset,
dt_asset,
dt_pivot_asset,
)
def run(db_url):
engine = Engine(db_url)
graph_posts_per_day = process_posts_per_day(engine)
post_per_day_caption = (
f"Number of posts updated per day (updated {get_current_time_in_vietname()})"
)
df_posts_per_domain = process_posts_per_domain(engine)
df_domain_sim, graph_domain_sim_score = process_sim_score_table(engine)
report = dp.Report(
dp.Plot(graph_posts_per_day, caption=post_per_day_caption),
dp.DataTable(df_posts_per_domain,
caption="Number of posts per domain"),
dp.DataTable(df_domain_sim, caption="Simplified similar_docs table"),
dp.Plot(graph_domain_sim_score,
caption="Similarity score histogram (>= 0.5)"),
)
report.publish(name="TopDup monitoring table", open=False)
print("View report at "
"https://datapane.com/u/tiepvupsu/reports/topdup-monitoring-table/")
def publish_report(df: pandas.DataFrame):
dp_token: str = db["DP_TOKEN"]
dp.login(token=dp_token)
plot = alt.Chart(df).mark_area(opacity=0.4, stroke='black').encode(
x='date:T',
y=alt.Y('new_cases_smoothed_per_million:Q', stack=None),
color=alt.Color('continent:N', scale=alt.Scale(scheme='set1')),
tooltip='continent:N').interactive().properties(width='container')
dp.Report(dp.Plot(plot), dp.DataTable(df)).save(path='report.html',open=True)
def gen_report_complex_with_files(datadir: Path,
single_file: bool = False,
local_report: bool = False) -> dp.Report:
# Asset tests
lis = [1, 2, 3]
small_df = gen_df()
big_df = gen_df(10000)
# text
# md_block
html_block = dp.HTML(html="<h1>Hello World</h1>")
html_block_1 = dp.HTML(html=h2("Hello World"))
code_block = dp.Code(code="print('hello')", language="python")
formula_block = dp.Formula(formula=r"\frac{1}{\sqrt{x^2 + 1}}")
big_number = dp.BigNumber(heading="Tests written", value=1234)
big_number_1 = dp.BigNumber(heading="Real Tests written :)",
value=11,
change=2,
is_upward_change=True)
embed_block = dp.Embed(url="https://www.youtube.com/watch?v=JDe14ulcfLA")
# assets
plot_asset = dp.Plot(data=gen_plot(), caption="Plot Asset")
list_asset = dp.File(data=lis, filename="List Asset", is_json=True)
img_asset = dp.File(file=datadir / "datapane-logo.png")
# tables
table_asset = dp.Table(data=small_df, caption="Test Basic Table")
# local reports don't support DataTable
dt_asset = table_asset if local_report else dp.DataTable(
df=big_df, caption="Test DataTable")
if single_file:
return dp.Report(dp.Group(blocks=[md_block, dt_asset]))
else:
return dp.Report(
dp.Page(
dp.Select(md_block,
html_block,
html_block_1,
code_block,
formula_block,
embed_block,
type=dp.SelectType.TABS),
dp.Group(big_number, big_number_1, columns=2),
),
dp.Page(
plot_asset,
list_asset,
img_asset,
table_asset,
dt_asset,
),
)
import os
import pandas as pd
import datapane as dp
# basic report creation, with params
df = pd.DataFrame.from_dict({"x": [4, 3, 2, 1], "y": [10.5, 20.5, 30.5, 40.5]})
blocks = [dp.Text(f"Dummy Markdown block - {dp.Params['p1']}"), dp.DataTable(df), dp.Text(f"Text block with env var: {os.environ['ENV_VAR']}")]
# test running as main or by datapane runner
if dp.ON_DATAPANE:
print("on datapane")
if __name__ == "__datapane__": # same as dp.by_datapane
print("by datapane")
report = dp.Report(blocks=blocks)
report.upload(name="dp_report", description="Description")
import pandas as pd
import datapane as dp
# basic report creation, with params
df = pd.DataFrame.from_dict({"x": [4, 3, 2, 1], "y": [10.5, 20.5, 30.5, 40.5]})
blocks = [dp.Markdown(f"Dummy Markdown block - {dp.Params['p1']}"), dp.DataTable(df)]
# test running as main or by datapane runner
if dp.on_datapane:
print("on datapane")
if __name__ == "__datapane__": # same as dp.by_datapane
print("by datapane")
report = dp.Report(blocks=blocks)
report.publish(name="dp_report", description="Description")
md_block = dp.Markdown(text="# Test markdown block \n Test **content**")
# In-line JSON
list_asset = dp.File(data=lis, is_json=True)
# Downloadable file
file_asset = dp.File(data=lis)
# In-line image
img_asset = dp.File(file=Path("./datapane-logo.png"))
# Vega
vega_asset = dp.Plot(data=alt.Chart(gen_df()).mark_line().encode(x="x", y="y"))
# Table
df_asset = dp.DataTable(df, can_pivot=False)
# Pivot table
pv_asset = dp.DataTable(gen_df(10), can_pivot=True)
# Matplotlib
np.random.seed(19680801)
xdata = np.random.random([2, 10])
xdata1 = xdata[0, :]
xdata2 = xdata[1, :]
xdata1.sort()
xdata2.sort()
ydata1 = xdata1 ** 2
ydata2 = 1 - xdata2 ** 3
mpl_fig = plt.figure(figsize=(15, 15))
ax = mpl_fig.add_subplot(1, 1, 1)
# In-line JSON
list_asset = dp.File(data=lis, is_json=True)
# Downloadable file
file_asset = dp.File(data=lis)
# In-line image
img_asset = dp.File(file=Path("./datapane-logo.png"))
# Vega
vega_asset = dp.Plot(data=gen_plot())
# Table
df_table_asset = dp.Table(gen_df())
df_datatable_asset = dp.DataTable(gen_df(10000))
# Matplotlib
np.random.seed(19680801)
xdata = np.random.random([2, 10])
xdata1 = xdata[0, :]
xdata2 = xdata[1, :]
xdata1.sort()
xdata2.sort()
ydata1 = xdata1**2
ydata2 = 1 - xdata2**3
mpl_fig = plt.figure(figsize=(15, 15))
ax = mpl_fig.add_subplot(1, 1, 1)
ax.plot(xdata1, ydata1, color='tab:blue')
ax.plot(xdata2, ydata2, color='tab:orange')
xevents1 = EventCollection(xdata1, color='tab:blue', linelength=0.05)
)
"""
r = dp.Report(
dp.Page(title="Dashes",
blocks=[
"#### Heatmap de Vagas pelo Brasil",
dp.Plot(mapa), "#### Total Vagas",
dp.Plot(fig3), "#### Total Vagas por Estado",
dp.Plot(fig1), "#### Total Vagas por Nível",
dp.Plot(fig2)
]),
dp.Page(title="Cientista de Dados",
blocks=[
"#### Vagas - Cientista de Dados",
dp.DataTable(df_CD[colunas], label="Cientista de Dados")
]),
dp.Page(title="Analista de Dados",
blocks=[
"#### Vagas Analista de Dados",
dp.DataTable(df_AD[colunas], label="Analista de Dados")
]),
dp.Page(title="Engenheiro de Dados",
blocks=[
"#### Vagas - Engenheiro de Dados",
dp.DataTable(df_ED[colunas], label="Engenheiro de Dados")
]),
dp.Page(title="Engenheiro de Machine Learning",
blocks=[
"#### Vagas - Engenheiro de Machine Learning",
dp.DataTable(df_EML[colunas],
from geo import m
from profiles import Arashiyama_df, Chayama_df, Demachiyanagi_df, Hanazono_df, Kokusaikaikan_df, izakaya_df, restaurants, station_groupby_df
# %% Report
rprt = dp.Report(
dp.Text("""
# 京都, in Stations and Their Restaurants
### Exploration of restaurants in Kyoto and the stations they're closest to
Splitting up Kyoto restaurants by regions centered on train/subway stations, as well as checking out some interesting characteristics of the different restaurants near each station. As it turns out, there's a ton of izakaya in the city
---
"""), dp.Plot(m),
dp.Text("""
## Dataset
The dataset used for the current project can be found [here](https://www.kaggle.com/koki25ando/tabelog-restaurant-review-dataset). Station data was pulled manually from wikipedia and google maps.
"""), dp.DataTable(restaurants),
dp.Text("""
## Some Special Station Profiles
*Taking a look at some stations which have interesting or peculiar restaurant selection*
"""), dp.DataTable(station_groupby_df),
dp.Text("""
### Arashiyama
#### Lots (relatively, at least) of tofu options, beware of expensive lunch prices
"""), dp.DataTable(Arashiyama_df),
dp.Text("""
### Demachiyanagi
#### A station whose food options are split between expensive kaiseki and cheap izakaya fare
"""), dp.DataTable(Demachiyanagi_df),
dp.Text("""
### Hanazono
#### Another station with polarized food options, with one shojin ryouri place and one meat-centric bbq joint
# pip install datapane
# pip install plotly
import datapane as dp
import pandas as pd
import numpy as np
import plotly.express as px
# Scripts to create df and chart
df = px.data.gapminder()
chart = px.scatter(df.query("year==2007"),
x="gdpPercap",
y="lifeExp",
size="pop",
color="continent",
hover_name="country",
log_x=True,
size_max=60)
# Once you have the df and the chart, simply use
r = dp.Report(
dp.Text("my simple report"), # add description
dp.DataTable(df), # create a table
dp.Plot(chart) # create a chart
)
# Publish your report
r.publish(name='example', visibility=dp.Visibility.PUBLIC)
change=7)
],
columns=2),
dp.Select(blocks=[
dp.Group(blocks=[
"""This plot models cash and revenue, dependent on various other growth scenarios and based on this month's burn.
- Blue area is cash remaining
- Orange line is the cumulative sum of all revenue
- Purple line is monthly revenue
> Drag the slider to adjust growth rate. The growth rate at which the blue area never crosses 0 is the growth you need to achieve **[Default Alive](http://paulgraham.com/aord.html)**.
""", chart_final
],
label='Interactive Plot'),
dp.Group(blocks=[
"> In this dataset, `cash_0g`/`revenue_0g` presumes no further growth, whereas `cash_dg`/`revenue_dg` presumes growth at the current rate continues.",
dp.DataTable(df)
],
label='Interactive Dataset')
])).publish(name='Finance Model', open=True)
"""# Deploying as an app
Because we defined the input variables using `dp.Params` above, we can also easily use Datapane to turn our script into an app which non-technical people can run themselves.
Note that deploying an app requires a private **Datapane Cloud** server, [so create one here](https://datapane.com/pricing/#contact-form) before continuing. Pricing starts at $99 per month with a 30-day free trial.
If we want user
To build an app, we need to include a YAML file which defines which parameters we want in our auto
"""
