def __init__(self,
num_rand_samples=1,
n_estimators=[8, 15],
max_depth=[10, 15],
n_jobs=-1,
tree_method='hist',
random_state=2,
seed=0,
lr=(1e-4, 1e-1),
subsample=0.8,
colsample_bytree=0.8,
min_child_weight=[1, 2, 3],
gamma=0,
reg_alpha=0,
reg_lambda=1):
"""
Constructor. For XGBoost hyper parameters, refer to
https://xgboost.readthedocs.io/en/latest/python/python_api.html for
details.
:param num_rand_samples: number of hyper-param configurations sampled
randomly
:param n_estimators: number of gradient boosted trees.
:param max_depth: max tree depth
:param n_jobs: number of parallel threads used to run xgboost.
:param tree_method: specify which tree method to use.
:param random_state: random number seed.
:param seed: seed used to generate the folds
:param lr: learning rate
:param subsample: subsample ratio of the training instance
:param colsample_bytree: subsample ratio of columns when constructing
each tree.
:param min_child_weight: minimum sum of instance weight(hessian)
needed in a child.
:param gamma: minimum loss reduction required to make a further
partition on a leaf node of the tree.
:param reg_alpha: L1 regularization term on weights (xgb’s alpha).
:param reg_lambda: L2 regularization term on weights (xgb’s lambda).
"""
super(self.__class__, self).__init__()
self.num_samples = num_rand_samples
self.n_jobs = n_jobs
self.tree_method = tree_method
self.random_state = random_state
self.seed = seed
self.colsample_bytree = colsample_bytree
self.gamma = gamma
self.reg_alpha = reg_alpha
self.reg_lambda = reg_lambda
self.n_estimators = hp.grid_search(n_estimators)
self.max_depth = hp.grid_search(max_depth)
self.lr = hp.loguniform(lr[0], lr[-1])
self.subsample = subsample
self.min_child_weight = hp.choice(min_child_weight)
def test_fit_lstm_data_creator(self):
input_feature_dim = 4
output_feature_dim = 2 # 2 targets are generated in get_tsdataset
search_space = {
'hidden_dim': hp.grid_search([32, 64]),
'layer_num': hp.randint(1, 3),
'lr': hp.choice([0.001, 0.003, 0.01]),
'dropout': hp.uniform(0.1, 0.2)
}
auto_estimator = AutoTSEstimator(model='lstm',
search_space=search_space,
past_seq_len=7,
future_seq_len=1,
input_feature_num=input_feature_dim,
output_target_num=output_feature_dim,
selected_features="auto",
metric="mse",
loss=torch.nn.MSELoss(),
logs_dir="/tmp/auto_trainer",
cpus_per_trial=2,
name="auto_trainer")
auto_estimator.fit(data=get_data_creator(),
epochs=1,
batch_size=hp.choice([32, 64]),
validation_data=get_data_creator(),
n_sampling=1)
config = auto_estimator.get_best_config()
assert config["past_seq_len"] == 7
def test_fit_third_party_data_creator(self):
input_feature_dim = 4
output_feature_dim = 2 # 2 targets are generated in get_tsdataset
search_space = {
'hidden_dim': hp.grid_search([32, 64]),
'dropout': hp.uniform(0.1, 0.2)
}
auto_estimator = AutoTSEstimator(model=model_creator,
search_space=search_space,
past_seq_len=7,
future_seq_len=1,
input_feature_num=input_feature_dim,
output_target_num=output_feature_dim,
selected_features="auto",
metric="mse",
loss=torch.nn.MSELoss(),
cpus_per_trial=2)
auto_estimator.fit(data=get_data_creator(),
epochs=1,
batch_size=hp.choice([32, 64]),
validation_data=get_data_creator(),
n_sampling=1)
config = auto_estimator.get_best_config()
assert config["past_seq_len"] == 7
def test_fit_third_party_feature(self):
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
tsdata_train = get_tsdataset().gen_dt_feature().scale(scaler, fit=True)
tsdata_valid = get_tsdataset().gen_dt_feature().scale(scaler,
fit=False)
search_space = {
'hidden_dim': hp.grid_search([32, 64]),
'dropout': hp.uniform(0.1, 0.2)
}
auto_estimator = AutoTSEstimator(model=model_creator,
search_space=search_space,
past_seq_len=hp.randint(4, 6),
future_seq_len=1,
selected_features="auto",
metric="mse",
loss=torch.nn.MSELoss(),
cpus_per_trial=2)
ts_pipeline = auto_estimator.fit(data=tsdata_train,
epochs=1,
batch_size=hp.choice([32, 64]),
validation_data=tsdata_valid,
n_sampling=1)
best_config = auto_estimator.get_best_config()
best_model = auto_estimator._get_best_automl_model()
assert 4 <= best_config["past_seq_len"] <= 6
assert isinstance(ts_pipeline, TSPipeline)
# use raw base model to predic and evaluate
tsdata_valid.roll(lookback=best_config["past_seq_len"],
horizon=0,
feature_col=best_config["selected_features"])
x_valid, y_valid = tsdata_valid.to_numpy()
y_pred_raw = best_model.predict(x_valid)
y_pred_raw = tsdata_valid.unscale_numpy(y_pred_raw)
# use tspipeline to predic and evaluate
eval_result = ts_pipeline.evaluate(tsdata_valid)
y_pred = ts_pipeline.predict(tsdata_valid)
# check if they are the same
np.testing.assert_almost_equal(y_pred, y_pred_raw)
# save and load
ts_pipeline.save("/tmp/auto_trainer/autots_tmp_model_3rdparty")
new_ts_pipeline = TSPipeline.load(
"/tmp/auto_trainer/autots_tmp_model_3rdparty")
# check if load ppl is the same as previous
eval_result_new = new_ts_pipeline.evaluate(tsdata_valid)
y_pred_new = new_ts_pipeline.predict(tsdata_valid)
np.testing.assert_almost_equal(eval_result[0], eval_result_new[0])
np.testing.assert_almost_equal(y_pred, y_pred_new)
# use tspipeline to incrementally train
new_ts_pipeline.fit(tsdata_valid)
def search_space(self):
return {
# -------- model selection TODO add MTNet
"model": hp.choice(["LSTM", "Seq2seq"]),
# --------- Vanilla LSTM model parameters
"lstm_1_units": hp.grid_search([16, 32]),
"dropout_1": 0.2,
"lstm_2_units": hp.grid_search([16, 32]),
"dropout_2": hp.uniform(0.2, 0.5),
# ----------- Seq2Seq model parameters
"latent_dim": hp.grid_search([32, 64]),
"dropout": hp.uniform(0.2, 0.5),
# ----------- optimization parameters
"lr": hp.uniform(0.001, 0.01),
"batch_size": hp.choice([32, 64]),
"epochs": self.epochs,
"past_seq_len": self.past_seq_config,
}
def __init__(self,
num_rand_samples=1,
epochs=5,
training_iteration=10,
look_back=2,
lstm_1_units=[16, 32, 64, 128],
lstm_2_units=[16, 32, 64],
batch_size=[32, 64]):
"""
__init__()
Constructor.
:param lstm_1_units: random search candidates for num of lstm_1_units
:param lstm_2_units: grid search candidates for num of lstm_1_units
:param batch_size: grid search candidates for batch size
:param num_rand_samples: number of hyper-param configurations sampled randomly
:param look_back: the length to look back, either a tuple with 2 int values,
which is in format is (min len, max len), or a single int, which is
a fixed length to look back.
:param training_iteration: no. of iterations for training (n epochs) in trials
:param epochs: no. of epochs to train in each iteration
"""
super(self.__class__, self).__init__()
# -- runtime params
self.num_samples = num_rand_samples
self.training_iteration = training_iteration
# -- model params
self.past_seq_config = PastSeqParamHandler.get_past_seq_config(
look_back)
self.lstm_1_units_config = hp.choice(lstm_1_units)
self.lstm_2_units_config = hp.grid_search(lstm_2_units)
self.dropout_2_config = hp.uniform(0.2, 0.5)
# -- optimization params
self.lr = hp.uniform(0.001, 0.01)
self.batch_size = hp.grid_search(batch_size)
self.epochs = epochs
def get_auto_estimator():
auto_lstm = AutoLSTM(input_feature_num=input_feature_dim,
output_target_num=output_feature_dim,
past_seq_len=5,
optimizer='Adam',
loss=torch.nn.MSELoss(),
metric="mse",
hidden_dim=hp.grid_search([32, 64]),
layer_num=hp.randint(1, 3),
lr=hp.choice([0.001, 0.003, 0.01]),
dropout=hp.uniform(0.1, 0.2),
logs_dir="/tmp/auto_lstm",
cpus_per_trial=2,
name="auto_lstm")
return auto_lstm
def __init__(self,
num_rand_samples=1,
training_iteration=40,
batch_size=[256, 512],
hidden_size=[32, 48],
levels=[6, 8],
kernel_size=[3, 5],
dropout=[0, 0.1],
lr=[0.001, 0.003]):
"""
__init__()
Constructor.
:param num_rand_samples: number of hyper-param configurations sampled randomly
:param training_iteration: no. of iterations for training (n epochs) in trials
:param batch_size: grid search candidates for batch size
:param hidden_size: grid search candidates for hidden size of each layer
:param levels: the number of layers
:param kernel_size: the kernel size of each layer
:param dropout: dropout rate (1 - keep probability)
:param lr: learning rate
"""
super(self.__class__, self).__init__()
# -- run time params
self.num_samples = num_rand_samples
self.training_iteration = training_iteration
# -- optimization params
self.lr = hp.choice(lr)
self.batch_size = hp.grid_search(batch_size)
# ---- model params
self.hidden_size = hp.grid_search(hidden_size)
self.levels = hp.grid_search(levels)
self.kernel_size = hp.grid_search(kernel_size)
self.dropout = hp.choice(dropout)
def _gen_sample_func(self, ranges, param_name):
if isinstance(ranges, tuple):
assert len(ranges) == 2, \
f"length of tuple {param_name} should be 2 while get {len(ranges)} instead."
assert param_name != "teacher_forcing", \
f"type of {param_name} can only be a list while get a tuple"
if param_name in ["lr"]:
return hp.loguniform(lower=ranges[0], upper=ranges[1])
if param_name in [
"lstm_hidden_dim", "lstm_layer_num", "batch_size"
]:
return hp.randint(lower=ranges[0], upper=ranges[1])
if param_name in ["dropout"]:
return hp.uniform(lower=ranges[0], upper=ranges[1])
if isinstance(ranges, list):
return hp.grid_search(ranges)
raise RuntimeError(f"{param_name} should be either a list or a tuple.")
def get_auto_estimator():
auto_seq2seq = AutoSeq2Seq(input_feature_num=input_feature_dim,
output_target_num=output_feature_dim,
past_seq_len=past_seq_len,
future_seq_len=future_seq_len,
optimizer='Adam',
loss=torch.nn.MSELoss(),
metric="mse",
lr=hp.choice([0.001, 0.003, 0.01]),
lstm_hidden_dim=hp.grid_search([32, 64, 128]),
lstm_layer_num=hp.randint(1, 4),
dropout=hp.uniform(0.1, 0.3),
teacher_forcing=False,
logs_dir="/tmp/auto_seq2seq",
cpus_per_trial=2,
name="auto_seq2seq")
return auto_seq2seq
def test_fit_data_creator(self):
auto_est = AutoEstimator.from_torch(model_creator=model_creator,
optimizer=get_optimizer,
loss=nn.BCELoss(),
logs_dir="/tmp/zoo_automl_logs",
resources_per_trial={"cpu": 2},
name="test_fit")
search_space = create_linear_search_space()
search_space.update({"shuffle": hp.grid_search([True, False])})
auto_est.fit(data=train_dataloader_creator,
validation_data=valid_dataloader_creator,
search_space=search_space,
n_sampling=2,
epochs=1,
metric="accuracy")
assert auto_est.get_best_model()
best_config = auto_est.get_best_config()
assert all(k in best_config.keys() for k in search_space.keys())
def test_select_feature(self):
sample_num = np.random.randint(100, 200)
df = pd.DataFrame({
"datetime":
pd.date_range('1/1/2019', periods=sample_num),
"value":
np.random.randn(sample_num),
"id":
np.array(['00'] * sample_num)
})
train_ts, val_ts, _ = TSDataset.from_pandas(df,
target_col=['value'],
dt_col='datetime',
id_col='id',
with_split=True,
val_ratio=0.1)
search_space = {
'hidden_dim': hp.grid_search([32, 64]),
'layer_num': hp.randint(1, 3),
'lr': hp.choice([0.001, 0.003, 0.01]),
'dropout': hp.uniform(0.1, 0.2)
}
input_feature_dim, output_feature_dim = 1, 1
auto_estimator = AutoTSEstimator(model='lstm',
search_space=search_space,
past_seq_len=6,
future_seq_len=1,
input_feature_num=input_feature_dim,
output_target_num=output_feature_dim,
selected_features="auto",
metric="mse",
loss=torch.nn.MSELoss(),
cpus_per_trial=2,
name="auto_trainer")
auto_estimator.fit(data=train_ts,
epochs=1,
batch_size=hp.choice([32, 64]),
validation_data=val_ts,
n_sampling=1)
config = auto_estimator.get_best_config()
assert config['past_seq_len'] == 6
def __init__(self,
num_rand_samples=1,
epochs=5,
training_iteration=10,
time_step=[3, 4],
long_num=[3, 4],
cnn_height=[2, 3],
cnn_hid_size=[32, 50, 100],
ar_size=[2, 3],
batch_size=[32, 64]):
"""
__init__()
Constructor.
:param num_rand_samples: number of hyper-param configurations sampled randomly
:param training_iteration: no. of iterations for training (n epochs) in trials
:param epochs: no. of epochs to train in each iteration
:param time_step: random search candidates for model param "time_step"
:param long_num: random search candidates for model param "long_num"
:param ar_size: random search candidates for model param "ar_size"
:param batch_size: grid search candidates for batch size
:param cnn_height: random search candidates for model param "cnn_height"
:param cnn_hid_size: random search candidates for model param "cnn_hid_size"
"""
super(self.__class__, self).__init__()
# -- run time params
self.num_samples = num_rand_samples
self.training_iteration = training_iteration
# -- optimization params
self.lr = hp.uniform(0.001, 0.01)
self.batch_size = hp.grid_search(batch_size)
self.epochs = epochs
# ---- model params
self.cnn_dropout = hp.uniform(0.2, 0.5)
self.rnn_dropout = hp.uniform(0.2, 0.5)
self.time_step = hp.choice(time_step)
self.long_num = hp.choice(long_num, )
self.cnn_height = hp.choice(cnn_height)
self.cnn_hid_size = hp.choice(cnn_hid_size)
self.ar_size = hp.choice(ar_size)
self.past_seq_len = hp.sample_from(
lambda spec: (spec.config.long_num + 1) * spec.config.time_step)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import bigdl.orca.automl.hp as hp
AUTO_MODEL_SUPPORT_LIST = ["lstm", "tcn", "seq2seq"]
AUTO_MODEL_DEFAULT_SEARCH_SPACE = {
"lstm": {"minimal": {"hidden_dim": hp.grid_search([16, 32]),
"layer_num": hp.randint(1, 2),
"lr": hp.loguniform(0.001, 0.005),
"dropout": hp.uniform(0.1, 0.2)},
"normal": {"hidden_dim": hp.grid_search([16, 32, 64]),
"layer_num": hp.grid_search([1, 2]),
"lr": hp.loguniform(0.0005, 0.01),
"dropout": hp.uniform(0, 0.2)},
"large": {"hidden_dim": hp.grid_search([16, 32, 64, 128]),
"layer_num": hp.grid_search([1, 2, 3, 4]),
"lr": hp.loguniform(0.0005, 0.01),
"dropout": hp.uniform(0, 0.3)}},
"tcn": {"minimal": {"hidden_units": hp.grid_search([16, 32]),
"levels": hp.randint(4, 6),
"kernel_size": 3,
def __init__(
self,
changepoint_prior_scale=hp.grid_search([0.005, 0.05, 0.1, 0.5]),
seasonality_prior_scale=hp.grid_search([0.01, 0.1, 1.0, 10.0]),
holidays_prior_scale=hp.loguniform(0.01, 10),
seasonality_mode=hp.choice(['additive', 'multiplicative']),
changepoint_range=hp.uniform(0.8, 0.95),
metric='mse',
logs_dir="/tmp/auto_prophet_logs",
cpus_per_trial=1,
name="auto_prophet",
remote_dir=None,
**prophet_config):
"""
Create an automated Prophet Model.
User need to specify either the exact value or the search space of the
Prophet model hyperparameters. For details of the Prophet model hyperparameters, refer to
https://facebook.github.io/prophet/docs/diagnostics.html#hyperparameter-tuning.
:param changepoint_prior_scale: Int or hp sampling function from an integer space
for hyperparameter changepoint_prior_scale for the Prophet model.
For hp sampling, see bigdl.chronos.orca.automl.hp for more details.
e.g. hp.loguniform(0.001, 0.5).
:param seasonality_prior_scale: hyperparameter seasonality_prior_scale for the
Prophet model.
e.g. hp.loguniform(0.01, 10).
:param holidays_prior_scale: hyperparameter holidays_prior_scale for the
Prophet model.
e.g. hp.loguniform(0.01, 10).
:param seasonality_mode: hyperparameter seasonality_mode for the
Prophet model.
e.g. hp.choice(['additive', 'multiplicative']).
:param changepoint_range: hyperparameter changepoint_range for the
Prophet model.
e.g. hp.uniform(0.8, 0.95).
:param metric: String. The evaluation metric name to optimize. e.g. "mse"
:param logs_dir: Local directory to save logs and results. It defaults to
"/tmp/auto_prophet_logs"
:param cpus_per_trial: Int. Number of cpus for each trial. It defaults to 1.
:param name: name of the AutoProphet. It defaults to "auto_prophet"
:param remote_dir: String. Remote directory to sync training results and checkpoints. It
defaults to None and doesn't take effects while running in local. While running in
cluster, it defaults to "hdfs:///tmp/{name}".
:param prophet_config: Other Prophet hyperparameters.
"""
self.search_space = {
"changepoint_prior_scale": changepoint_prior_scale,
"seasonality_prior_scale": seasonality_prior_scale,
"holidays_prior_scale": holidays_prior_scale,
"seasonality_mode": seasonality_mode,
"changepoint_range": changepoint_range
}
self.search_space.update(prophet_config) # update other configs
self.metric = metric
model_builder = ProphetBuilder()
self.auto_est = AutoEstimator(
model_builder=model_builder,
logs_dir=logs_dir,
resources_per_trial={"cpu": cpus_per_trial},
remote_dir=remote_dir,
name=name)
search_alg_params = dict(space=space,
name=experiment_name,
max_concurrent=1)
search_space = dict()
search_alg = "sigopt"
search_alg_params = search_alg_params
scheduler = "AsyncHyperBand"
scheduler_params = dict(
max_t=50,
grace_period=1,
reduction_factor=3,
brackets=3,
)
else:
search_space = {
"n_estimators": hp.grid_search(list(n_estimators_range)),
"max_depth": hp.grid_search(list(max_depth_range)),
"lr": hp.loguniform(1e-4, 1e-1),
"min_child_weight": hp.choice(min_child_weight),
}
search_alg = None
search_alg_params = None
scheduler = None
scheduler_params = None
auto_xgb_reg = AutoXGBRegressor(cpus_per_trial=2,
name="auto_xgb_regressor",
**config)
auto_xgb_reg.fit(data=(X_train, y_train),
validation_data=(X_val, y_val),
metric="rmse",
help="Max number of epochs to train in each trial.")
args = parser.parse_args()
num_nodes = 1 if args.cluster_mode == "local" else args.num_workers
init_orca_context(cluster_mode=args.cluster_mode,
cores=args.cores,
memory=args.memory,
num_nodes=num_nodes,
init_ray_on_spark=True)
tsdata_train, tsdata_valid, tsdata_test = get_tsdata()
auto_lstm = AutoLSTM(input_feature_num=1,
output_target_num=1,
past_seq_len=20,
hidden_dim=hp.grid_search([32, 64]),
layer_num=hp.randint(1, 3),
lr=hp.choice([0.01, 0.03, 0.1]),
dropout=hp.uniform(0.1, 0.2),
optimizer='Adam',
loss=torch.nn.MSELoss(),
metric="mse")
x_train, y_train = tsdata_train.roll(lookback=20, horizon=1).to_numpy()
x_val, y_val = tsdata_test.roll(lookback=20, horizon=1).to_numpy()
x_test, y_test = tsdata_test.roll(lookback=20, horizon=1).to_numpy()
auto_lstm.fit(data=(x_train, y_train),
epochs=args.epochs,
validation_data=(x_val, y_val))