Python Thunderbolt Examples

Example #1

File: test_local_cache_regression.py Project: m3dev/thunderbolt

    def test_running(self):
        target = Thunderbolt(self._get_test_case_path(), use_cache=False)
        _ = Thunderbolt(self._get_test_case_path())
        output = Thunderbolt(self._get_test_case_path())

        for k, v in target.tasks.items():
            if k == 'last_modified':  # cache file
                continue
            self.assertEqual(v, output.tasks[k])

        output.client.local_cache.clear()

Example #2

File: events.py Project: kiccho1101/kaggle_m5_forecasting

# %%
from thunderbolt import Thunderbolt
import pandas as pd
import sys
import os
import matplotlib.pyplot as plt

sys.path.append(os.getcwd() + "/../..")

from kaggle_m5_forecasting.data.load_data import RawData
from kaggle_m5_forecasting.utils import decode_ids

tb = Thunderbolt("./../../resource")
data: pd.DataFrame = tb.get_data("MakeData")
data = decode_ids(data)
raw: RawData = tb.get_data("LoadRawData")

# %%
df = data.groupby("id")[["sell_price", "sales"]].mean().reset_index()

# %%

sell_price = data.groupby("item_id")["sell_price"].mean()

d_cols = [f"d_{d}" for d in range(1, 1942)]
date_idx = pd.to_datetime(raw.calendar["date"][:1941])
for i, s in enumerate(
        raw.sales_train_validation[raw.sales_train_validation.cat_id ==
                                   "FOODS"][["item_id"] + d_cols].groupby(
                                       "item_id").sum().reset_index().values):
    if i >= 150 and i <= 180:

Example #3

File: dev.py Project: kiccho1101/kaggle_disaster_tweets_gokart

import luigi
import numpy as np
import pandas as pd

import kaggle_disaster_tweets_gokart

luigi.configuration.LuigiConfigParser.add_config_path("./conf/param.ini")
np.random.seed(42)

# %%
# gokart.run(["tweet.MakeEnsembleModel", "--rerun"])

# %%
from thunderbolt import Thunderbolt

tb = Thunderbolt("./resource")
tb.get_task_df()

# %%
import numpy as np

# %%
import re
import regex
import swifter

df: pd.DataFrame = tb.get_data("MakeTrainSelectedFeatureData")

# %%

Example #4

# %%
import numpy as np

import sys
import os

sys.path.append(os.getcwd() + "/../..")
from kaggle_m5_forecasting.cv_result import CVResults
from kaggle_m5_forecasting.data.load_data import RawData
from kaggle_m5_forecasting.cv_dashboard import create_dashboard
from thunderbolt import Thunderbolt

TIMESTAMP = "2020-06-18_23:14:13"

cv = CVResults().from_timestamp(TIMESTAMP)
tb = Thunderbolt("./../../resource")
raw: RawData = tb.get_data("LoadRawData")

# %%
CV_NUM = 0
dir_name = f"../../output/cv/{TIMESTAMP}/{CV_NUM}"
evaluator = cv.results[CV_NUM].get_evaluator(raw)
# create_dashboard(
#     evaluator, raw, dir_name,
# )
print(np.mean(evaluator.all_scores))

Example #5

File: weather.py Project: kiccho1101/kaggle_m5_forecasting

# %%
import pandas as pd
from thunderbolt import Thunderbolt
import sys
import os

sys.path.append(os.getcwd() + "/../..")

from kaggle_m5_forecasting.data.load_data import RawData
from kaggle_m5_forecasting.data.fe_weather import read_weather_data
from kaggle_m5_forecasting.utils import decode_ids

tb = Thunderbolt("./../../resource")
data: pd.DataFrame = pd.concat(
    [tb.get_data("MakeData"),
     tb.get_data("FEWeather")], axis=1)
weather = read_weather_data("./../../external_data")
weather["date"] = pd.to_datetime(weather["date_time"]).dt.strftime("%Y-%m-%d")
weather.index = pd.to_datetime(weather["date_time"])
weather.index.name = None
weather.drop("date_time", axis=1, inplace=True)

raw: RawData = tb.get_data("LoadRawData")

# %%
raw.calendar["d"] = raw.calendar["d"].apply(lambda d: int(d.replace("d_", "")))
cat_id = 0
df: pd.DataFrame = data[data["cat_id"] == cat_id].groupby(
    ["d", "state_id"])["sales"].mean().reset_index().merge(
        raw.calendar[["d", "date"]], on="d",
        how="left").merge(weather, on=["date", "state_id"], how="left")

Example #6

File: clustering.py Project: kiccho1101/kaggle_m5_forecasting

import matplotlib.pyplot as plt
import sklearn.metrics
import catch22
import seaborn as sns
from thunderbolt import Thunderbolt
import scipy
import sklearn.preprocessing
import sklearn.cluster

import sys
import os

sys.path.append(os.getcwd() + "/../..")
from kaggle_m5_forecasting.utils import timer

tb = Thunderbolt("./../../resource")
data: pd.DataFrame = tb.get_data("MakeData")
data = data[data.d < 1942]

# %%

with timer("calc grouped aggregates"):
    grouped = data.groupby(["id"])["sales"].agg({
        "mean":
        lambda x: x.dropna().values.mean(),
        "percentile25":
        lambda x: x.dropna().sort_values()[:int(len(x) * 0.25)].mean(),
        "percentile50":
        lambda x: x.dropna().sort_values()[int(len(x) * 0.25):int(
            len(x) * 0.5)].mean(),
        "percentile75":

Example #7

# %%
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from thunderbolt import Thunderbolt
import sys
import os

sys.path.append(os.getcwd() + "/../..")
from kaggle_m5_forecasting.utils import timer

tb = Thunderbolt("./../../resource")
data: pd.DataFrame = tb.get_data("MakeData")

# %%

with timer("calc rolling_store_id_cat_id_mean"):
    lag = 28
    w_size = 30
    data["fe_rolling_store_id_cat_id_mean"] = data.groupby([
        "store_id", "cat_id"
    ])["sales"].transform(lambda x: x.shift(lag).rolling(w_size).mean())

# %%
tb = Thunderbolt("./../../resource")
tb.get_data("FERollingGroupMean")