def run():
# torch.multiprocessing.freeze_support()
optimizer = Optimizer(optimizer_config)
for parameters in optimizer.get_parameters():
hyperparameters = Namespace(**parameters["parameters"])
model = PyTorchLightningModel(hparams=hyperparameters)
comet_logger = CometLogger(
api_key=get_config("comet.api_key"),
rest_api_key=get_config("comet.api_key"),
optimizer_data=parameters,
)
trainer = Trainer(
max_epochs=1,
# early_stop_callback=True, # requires val_loss be logged
logger=[comet_logger],
# num_processes=2,
# distributed_backend='ddp_cpu'
)
trainer.fit(model)
def run_optimizer():
optimizer = Optimizer(API_KEY)
params = """
epochs integer [5, 10] [5]
batch_size integer [5000, 15000] [10000]
weight_regularizer real [1e-5, 1e-2] [1e-3]
"""
optimizer.set_params(params)
# get_suggestion will raise when no new suggestion is available
i = 0
while True:
# Get a suggestion
suggestion = optimizer.get_suggestion()
print()
print('----------------------------------')
print(i)
print(suggestion)
print('batch_size', suggestion['batch_size'])
print('epochs', suggestion['epochs'])
print('weight_regularizer', suggestion['weight_regularizer'])
print('----------------------------------')
print()
# Create a new experiment associated with the Optimizer
experiment = Experiment(api_key=API_KEY, project_name="consensusnet")
score = train_with_optimizer(suggestion, experiment)
# Report the score back
suggestion.report_score("accuracy", score)
def main():
global model_def
global dataset
global train
args = argument_parser()
config_module = importlib.import_module(args.config)
configs = config_module.config
model_def = importlib.import_module(args.model).model
dataset = importlib.import_module(args.dataset).dataset
train = importlib.import_module(args.trainer).train
if 'algorithm' in configs.keys():
comet_config = parse_comet_config(configs)
opt = Optimizer(comet_config,
api_key=configs['API_KEY'],
project_name=configs['project_name'])
for exp in opt.get_experiments():
experiment = exp
config = get_parameters(experiment, configs)
train(
**{
'config': config,
'model_def': model_def,
'dataset': dataset,
'experiment': experiment
})
else:
if args.experiment:
experiment = Experiment(api_key=configs['API_KEY'],
project_name=configs['project_name'],
workspace=configs['workspace'])
else:
experiment = None
tried_configs = []
end = False
while True:
importlib.reload(config_module)
configs = config_module.config
possible_configs = get_configurations(configs)
for config_idx, config in enumerate(possible_configs):
if config_idx == len(possible_configs) - 1:
end = True
if config in tried_configs:
continue
else:
tried_configs.append(config)
train(
**{
'config': config,
'model_def': model_def,
'dataset': dataset,
'experiment': experiment
})
break
if end:
break
print("******************End of the training session****************")
def run_optimizer():
optimizer = Optimizer(API_KEY)
params = """
epochs integer [15, 100] [20]
batch_size integer [5000, 15000] [10000]
filters_1 integer [30, 50] [40]
filters_2 integer [30, 50] [40]
filters_3 integer [30, 50] [40]
kernel_size_1 integer [3, 10] [3]
kernel_size_2 integer [3, 10] [3]
kernel_size_3 integer [3, 10] [3]
pool_size_1 integer [2, 4] [2]
pool_size_2 integer [2, 4] [2]
weight_regularizer real [1e-5, 1e-2] [1e-3]
"""
optimizer.set_params(params)
# get_suggestion will raise when no new suggestion is available
i = 0
while True:
i += 1
# Get a suggestion
suggestion = optimizer.get_suggestion()
print()
print('----------------------------------')
print(i)
print('batch_size', suggestion['batch_size'])
print('epochs', suggestion['epochs'])
print('filters_1', suggestion['filters_1'])
print('filters_2', suggestion['filters_2'])
print('filters_3', suggestion['filters_3'])
print('kernel_size_1', suggestion['kernel_size_1'])
print('kernel_size_2', suggestion['kernel_size_2'])
print('kernel_size_3', suggestion['kernel_size_3'])
print('pool_size_1', suggestion['pool_size_1'])
print('pool_size_2', suggestion['pool_size_2'])
print('weight_regularizer', suggestion['weight_regularizer'])
print('----------------------------------')
print()
# Create a new experiment associated with the Optimizer
experiment = Experiment(api_key=API_KEY, project_name="consensusnet")
score = train_with_optimizer(suggestion, experiment)
# Report the score back
suggestion.report_score("accuracy", score)
def test_config_gen_2():
if _COMET_AVAILABLE:
config = TunerConfigGenerator()
config.suggest_categorical("categorical", ["a", "b", "c"])
config.suggest_uniform("uniform", 0.0, 1.0)
config.suggest_normal("normal", 0.0, 1.0)
config.suggest_lognormal("lognormal", 0.0, 1.0)
config.suggest_loguniform("loguniform", 0.0, 1.0)
config.suggest_discrete("discrete", [10, 20, 30])
config.export("test.config")
is_pass = False
try:
opt = Optimizer("test.config", experiment_class="OfflineExperiment")
except exceptions.OptimizerException as e:
print(e)
is_pass = False
except Exception as e:
print(e)
else:
is_pass = True
assert is_pass
else:
assert True
def main():
num_classes = 10
# the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype("float32")
x_test = x_test.astype("float32")
x_train /= 255
x_test /= 255
print(x_train.shape[0], "train samples")
print(x_test.shape[0], "test samples")
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
config = {
"algorithm": "bayes",
"name": "Optimize MNIST Network",
"spec": {
"maxCombo": 10,
"objective": "minimize",
"metric": "loss"
},
"parameters": {
"first_layer_units": {
"type": "integer",
"min": 1,
"max": 1000
}
},
"trials": 1,
}
opt = Optimizer(config)
for experiment in opt.get_experiments():
flu = experiment.get_parameter("first_layer_units")
loss = fit(experiment, x_train, y_train, x_test, y_test, 3, 120, flu)
# Reverse the score for minimization
experiment.log_metric("loss", loss)
def main():
num_classes = 10
# the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype("float32")
x_test = x_test.astype("float32")
x_train /= 255
x_test /= 255
print(x_train.shape[0], "train samples")
print(x_test.shape[0], "test samples")
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
## Gets API from config or environment:
opt = Optimizer()
pcs_content = """
first_layer_units integer [1,1000] [2]
"""
# opt.set_params(pcs_content)
opt.set_params(pcs_content)
while True:
try:
sug = opt.get_suggestion()
except NoMoreSuggestionsAvailable:
break
print("SUG", sug, sug.__dict__)
flu = sug["first_layer_units"]
print("FLU", repr(flu))
score = train(x_train, y_train, x_test, y_test, 3, 120, flu)
print("Score", score, sug.__dict__)
# Reverse the score for minimization
sug.report_score("score", score)
def main(cfg: DictConfig):
train_length = 500
val_length = 30
test_length = 585
opt = Optimizer(
config,
api_key='SWhvV0XPkHV8tPdU8Nv67EXxU',
project_name=f'stock-gcn-experiment-sequences-{train_length}days')
for experiment in opt.get_experiments():
seed = 1
seed_everything(seed)
hidden_layer = experiment.get_parameter('hidden_layer')
hidden_feature = experiment.get_parameter('hidden_feature')
activation = experiment.get_parameter('activation')
sequence_length = experiment.get_parameter('sequence_length')
KOSPI200Dataset.setup(train_length=train_length,
val_length=val_length,
test_length=test_length,
sequence_length=sequence_length
) # 2020.07.03 615일(개장일 기준) 이전이 2018.01.02
comet_logger = CometLogger(
api_key="SWhvV0XPkHV8tPdU8Nv67EXxU",
workspace="dldjwls3", # Optional
project_name=f'stock-gcn-experiment-sequences-{train_length}days',
experiment_name=
f'gcn_{sequence_length}_{activation}_{hidden_layer}_{hidden_feature}',
experiment_key=experiment.get_key())
model = Baseline(seed=seed,
sequence_length=sequence_length,
num_feature=5,
hidden_layer=hidden_layer,
hidden_feature=hidden_feature,
activation=activation)
trainer = Trainer(max_epochs=120, gpus=-1, logger=comet_logger)
trainer.fit(model)
def run_logistic_regression(train_df, validation_df):
params = """
C real [0.00001, 0.0001] [0.0001]
"""
optimizer = Optimizer(API_KEY)
optimizer.set_params(params)
while True:
suggestion = optimizer.get_suggestion()
experiment = Experiment(api_key=API_KEY, project_name='home-credit')
experiment.set_name('logreg')
experiment.log_dataset_hash(
pd.concat([train_df, validation_df], axis=0))
experiment.log_parameter(name='C', value=suggestion['C'])
logreg = LogisticRegression(C=suggestion['C'])
logreg.fit(train_df.drop(columns=['TARGET']), train_df["TARGET"])
y_pred = logreg.predict(validation_df.drop(columns=['TARGET']))
auc_score = roc_auc_score(validation_df['TARGET'], y_pred)
experiment.log_metric(name='auc_score', value=auc_score)
suggestion.report_score("auc_score", auc_score)
def run_optimizer(args):
optimizer = Optimizer(API_KEY)
params = """
epochs integer [5, 10] [5]
batch_size integer [64, 256] [64]
learning_rate real [0.0001, 0.01] [0.0001]
embedding_dimension integer [25, 200] [25]
"""
optimizer.set_params(params)
# get_suggestion will raise when no new suggestion is available
while True:
# Get a suggestion
suggestion = optimizer.get_suggestion()
# Create a new experiment associated with the Optimizer
experiment = Experiment(
api_key=API_KEY, project_name="fasttext")
score = train_with_optimizer(suggestion, experiment, args)
# Report the score back
suggestion.report_score("accuracy", score)
def run_lightgbm(train_df, validation_df):
train_data = lgb.Dataset(data=train_df.drop(columns=['TARGET']),
label=train_df['TARGET'])
validation_data = lgb.Dataset(data=validation_df.drop(columns=['TARGET']),
label=validation_df['TARGET'])
num_round = 10
params = """
num_leaves integer [31, 51] [31]
num_trees integer [50, 100] [50]
"""
optimizer = Optimizer(API_KEY)
optimizer.set_params(params)
while True:
suggestion = optimizer.get_suggestion()
experiment = Experiment(api_key=API_KEY, project_name='home-credit')
experiment.set_name('lightgbm')
_param = {
'num_leaves': suggestion['num_leaves'],
'num_trees': suggestion['num_trees'],
'objective': 'binary',
'metric': 'auc'
}
experiment.log_multiple_params(_param)
experiment.log_dataset_hash(
pd.concat([train_df, validation_df], axis=0))
bst = lgb.train(_param,
train_data,
num_round,
valid_sets=[validation_data])
y_pred = bst.predict(validation_df.drop(columns=['TARGET']))
auc_score = roc_auc_score(validation_df['TARGET'], y_pred)
experiment.log_metric(name='auc_score', value=auc_score)
suggestion.report_score("auc_score", auc_score)
def run_random_forest(train_df, validation_df):
params = """
n_estimators integer [100, 500] [100]
"""
optimizer = Optimizer(API_KEY)
optimizer.set_params(params)
while True:
suggestion = optimizer.get_suggestion()
experiment = Experiment(api_key=API_KEY, project_name='home-credit')
experiment.log_dataset_hash(
pd.concat([train_df, validation_df], axis=0))
experiment.set_name('rf')
experiment.log_parameter(name='n_estimators',
value=suggestion['n_estimators'])
rf = RandomForestClassifier(n_estimators=suggestion['n_estimators'])
rf.fit(train_df.drop(columns=['TARGET']), train_df["TARGET"])
y_pred = rf.predict(validation_df.drop(columns=['TARGET']))
auc_score = roc_auc_score(validation_df['TARGET'], y_pred)
experiment.log_metric(name='auc_score', value=auc_score)
suggestion.report_score("auc_score", auc_score)
def optimize(max_n,
model_name,
dataset_name,
batch_size,
n_epochs,
use_comet,
comet_name,
model_module_index=None):
dataset_config, model = initialize_model(dataset_name, model_name,
model_module_index)
rocket = None
#if use_comet:
# rocket = CometConnection(comet_name=comet_name, dataset_config=dataset_config)
# TODO: add the optimizer code to comet_connection.py
params_range = get_params_range(model)
params_range['spec']['maxCombo'] = max_n
optimizer = Optimizer(params_range, api_key=COMET_KEY)
for experiment in optimizer.get_experiments(project_name=PROJECT_NAME):
experiment.set_name(comet_name)
experiment.add_tag("optimizer_experiment")
model_exp = model
p = {
k: experiment.get_parameter(k)
for k in params_range['parameters'].keys()
}
model_exp.params = p
model_exp = train_model(model,
dataset_name,
dataset_config,
batch_size,
n_epochs,
compile_dict=COMPILE_DICT)
loss = model_exp.test_results[0]
experiment.log_metric("loss", loss)
def optimize_hyper_param(self, main_config, opt_config_file_path):
"""
Optimizer model hyper-parameters and upload results to cloud.
"""
self.opt_config_file_path = opt_config_file_path
print("# load model-optimization config")
config = load_config(config_file_path=self.opt_config_file_path,
purpose="optimize_tree")
element.pop("metric", None)
self.opt = Optimizer(
config,
api_key="VNQSdbR1pw33EkuHbUsGUSZWr",
project_name="general",
workspace="florpi",
experiment_class="OfflineExperiment",
#offline_directory="/cosma/home/dp004/dc-beck3/4_GaHaCo/GaHaCo/comet/",
offline_directory="/cosma/home/dp004/dc-cues1/GaHaCo/comet/",
)
#"Layer_ints":{"type":"integer","min":2,"max":6}
#"activation":{"type": "categorical","values":["elu","sigmoid"]}
},
# Declare what we will be optimizing, and how:
"spec": {
"metric": "ROC",
"objective": "maximize",
},
}
parameters = open("parameters.yml")
yamlparameters = yaml.load(parameters, Loader=yaml.FullLoader)
opt = Optimizer(config,
api_key=yamlparameters["comet_api_key"],
project_name="NNqhmv6",
auto_metric_logging=True)
X_train, X_test, y_train, y_test = get_features(yamlparameters["DataDir"])
for experiment in opt.get_experiments():
keras_model = models.qdense_model(
Input(shape=X_train.shape[1:]),
l1Reg=experiment.get_parameter("Regularization"),
bits=14,
ints=2)
#keras_model = models.dense_model(Input(shape=X_train.shape[1:]), l1Reg=experiment.get_parameter("Regularization"))
startlearningrate = experiment.get_parameter("learning_rate")
adam = Adam(lr=startlearningrate,
beta_1=experiment.get_parameter("learning_beta1"),
beta_2=experiment.get_parameter("learning_beta2"),
import torch.nn.functional as F
from sklearn import metrics
# Constant to modify
PROJ_LOCATION = '/vol/ml/xduan7/Projects/MoReL'
DATA_LOCATION = '/vol/ml/xduan7/Data'
import sys
sys.path.extend([PROJ_LOCATION])
from utils.dataset.drug_resp_dataset import *
from network.gnn.gat.gat import EdgeGATEncoder
from network.gnn.gcn.gcn import EdgeGCNEncoder
from network.gnn.mpnn.mpnn import MPNN
from network.simple_uno import SimpleUno
comet_opt = Optimizer(project_name='Drug Response with Graph Models')
# Construct the datasets for training and testing
bigrun_cell_id_list = pd.read_csv(DATA_LOCATION + '/bigrun_cell_ids.csv',
index_col=None).values.reshape(
(-1)).tolist()
bigrun_drug_id_list = pd.read_csv(DATA_LOCATION + '/bigrun_drug_ids.csv',
index_col=None).values.reshape(
(-1)).tolist()
trn_dset, tst_dset, _, _ = get_datasets(
resp_data_path=(DATA_LOCATION +
'/combined_single_drug_response_aggregated.csv'),
resp_aggregated=True,
resp_target='AUC',
resp_data_sources=DATA_SOURCES,
def train(X_train=None,
Y_train=None,
validation_data=(None, None),
batch=None,
epochs=None):
model = build_model()
model.compile(optimizer=Adam(), loss="mse", metrics=["accuracy"])
results = model.fit(X_train,
Y_train,
batch_size=batch,
epochs=epochs,
validation_data=validation_data)
return results
opt = Optimizer(sys.argv[1],
workspace="returncode13",
project_name="UNET-training-01")
(X, Y, X_val, Y_val) = create_data()
for experiment in opt.get_experiments():
results = train(X,
Y,
batch=experiment.get_parameter('batch'),
epochs=100,
validation_data=(X_val, Y_val))
experiment.log_metric('loss', results.history['loss'])
experiment.log_metric('accuracy', results.history['accuracy'])
def optimise(self):
config = {
# We pick the Bayes algorithm:
"algorithm": "bayes",
# Declare your hyperparameters in the Vizier-inspired format:
"parameters": {
"max_depth": {
"type": "integer",
"min": self.max_depth["min"],
"max": self.max_depth["max"],
"scalingType": "uniform"
},
"learning_rate": {
"type": "float",
"min": self.learning_rate["min"],
"max": self.learning_rate["max"],
"scalingType": "uniform"
},
"gamma": {
"type": "float",
"min": self.gamma["min"],
"max": self.gamma["max"],
"scalingType": "uniform"
},
"subsample": {
"type": "float",
"min": self.subsample["min"],
"max": self.subsample["max"],
"scalingType": "uniform"
},
"min_child_weight": {
"type": "float",
"min": self.min_child_weight["min"],
"max": self.min_child_weight["max"],
"scalingType": "uniform"
},
"reg_alpha": {
"type": "float",
"min": self.alpha["min"],
"max": self.alpha["max"],
"scalingType": "uniform"
},
"early_stopping": {
"type": "integer",
"min": self.early_stopping["min"],
"max": self.early_stopping["max"],
"scalingType": "uniform"
},
"rate_drop": {
"type": "float",
"min": self.rate_drop["min"],
"max": self.rate_drop["max"],
"scalingType": "uniform"
},
"skip_drop": {
"type": "float",
"min": self.skip_drop["min"],
"max": self.skip_drop["max"],
"scalingType": "uniform"
}
},
# Declare what we will be optimizing, and how:
"spec": {
"metric": "ROC",
"objective": "maximize",
},
}
opt = Optimizer(config,
api_key=self.comet_api_key,
project_name=self.comet_project_name,
auto_metric_logging=True)
for experiment in opt.get_experiments():
self.param["learning_rate"] = experiment.get_parameter(
"learning_rate")
self.num_rounds = self.n_estimators
self.param["subsample"] = experiment.get_parameter("subsample")
self.param["max_depth"] = experiment.get_parameter("max_depth")
self.param["gamma"] = experiment.get_parameter("gamma")
self.param["reg_alpha"] = experiment.get_parameter("reg_alpha")
self.param["min_child_weight"] = experiment.get_parameter(
"min_child_weight")
self.param["rate_drop"] = experiment.get_parameter("rate_drop")
self.param["skip_drop"] = experiment.get_parameter("skip_drop")
self.early_stopping_rounds = experiment.get_parameter(
"early_stopping")
self.train()
self.test()
auc, binary_accuracy = self.evaluate()
experiment.log_metric("ROC", auc)
experiment.log_metric("Binary_Accuracy", binary_accuracy)
experiment.log_metric("Best Boost Round", self.boost_rounds)
experiment.log_metric("score", (auc / 0.5 - 1) +
(1 - self.boost_rounds / self.n_estimators))
'to_tokens': False,
'embedding_dim': 50,
'embedding_type': 'glove',
'embedding_source': 'glove.6B.50d'}
# Task and experiment name
task_name = experiment_params['task_name']
experiment_name = experiment_params['experiment_name']
# Load optimiser config
optimiser_config_file = 'model_optimiser_configs.json'
with open(optimiser_config_file) as json_file:
optimiser_config = json.load(json_file)[experiment_params['model_name']]
# Set up comet optimiser
model_optimiser = Optimizer(optimiser_config)
# Data set and output paths
dataset_name = 'token_dataset' if experiment_params['to_tokens'] else 'text_dataset'
dataset_dir = os.path.join(task_name, dataset_name)
embeddings_dir = 'embeddings'
# Create appropriate directories if they don't exist
for directory in [task_name, dataset_dir, embeddings_dir]:
if not os.path.exists(directory):
os.mkdir(directory)
print("------------------------------------")
print("Running experiment...")
print(task_name + ": " + experiment_name)
import torch.optim as optim
from sklearn.datasets import load_iris
import enchanter.tasks as tasks
import enchanter.addons as addons # pylint: disable=W0611
import enchanter.addons.layers as layers
from enchanter.utils import comet
config = comet.TunerConfigGenerator(
algorithm="bayes",
metric="train_avg_loss",
objective="minimize",
seed=0,
trials=5
)
config.suggest_categorical("activation", ["addons.mish", "torch.relu", "torch.sigmoid"])
opt = Optimizer(config.generate())
for experiment in opt.get_experiments():
model = layers.MLP([4, 512, 128, 3], eval(experiment.get_parameter("activation")))
optimizer = optim.Adam(model.parameters())
runner = tasks.ClassificationRunner(
model, optimizer=optimizer, criterion=nn.CrossEntropyLoss(), experiment=experiment
)
x, y = load_iris(return_X_y=True)
x = x.astype("float32")
y = y.astype("int64")
runner.fit(x, y, epochs=1, batch_size=32)
def score_bayes_trainer_harness(self):
"""
Returns
-------
:return: ``None``
None, but results in saved models suitable for scoring and trained
on all available data.
"""
self.__log.info("Starting generic score train loop")
train_data = self.train_data
val_data = self.val_data
models = self.model
out_path = self.out_path
type_time_series = self.type_time_series
param_search = self.param_search
trainer = self.trainer
api_key = self.api_key
rest_api_key = self.rest_api_key
workspace = self.workspace
for models_ in models.get(type_time_series):
for key, value in models_.items():
model_name = key
model = value[0]
model_kwarg = value[1]
if param_search == 'bayes':
search_space = GluonTSBayesEstimatorKwargs.BayesModelLookup.get(
model_name)
# comet-ml hyperparameter optimization configuration (bayes in this case)
config = {
"algorithm": "bayes",
"spec": {
"maxCombo": 5, # no of combinations to try
"objective": "minimize",
"metric": "loss",
"seed": 42,
"gridSize": 10,
"minSampleSize": 100,
"retryLimit": 20,
"retryAssignLimit": 0,
},
"name": "My Bayesian Search",
"trials": 1,
}
config['parameters'] = search_space
# current time
timestr = time.strftime("%Y%m%d-%H%M%S")
# comet-ml project name for the optimization
project_name = f"optimizer-{model_name}-{timestr}"
# initialize the comet-ml optimizer
optimizer = Optimizer(config=config,
api_key=api_key,
project_name=project_name)
# loop through the parameter combinations that the bayes optimizer suggests
for experiment in optimizer.get_experiments():
# explicitly set the model parameters (should be generic for any model)
if model_name == "SimpleFeedForward":
hidden1 = experiment.get_parameter(
"hidden_layer_size")
hidden2 = experiment.get_parameter(
"hidden2_layer_size")
model_kwarg['num_hidden_dimensions'] = [
hidden1, hidden2
]
self.__log.info(
f"model_kwarg['num_hidden_dimensions'] : {model_kwarg['num_hidden_dimensions']}"
)
elif model_name == "DeepAREstimate":
model_kwarg[
'num_layers'] = experiment.get_parameter(
"num_layers")
model_kwarg[
'num_cells'] = experiment.get_parameter(
"num_cells")
model_kwarg[
'cell_type'] = experiment.get_parameter(
"cell_type")
model_kwarg[
'dropout_rate'] = experiment.get_parameter(
"dropout_rate")
# set trainer params
trainer.learning_rate = experiment.get_parameter(
"learning_rate")
trainer.batch_size = experiment.get_parameter(
"batch_size")
trainer.epochs = 2
# initialize model from the suggested hyperparameters
model = model.from_hyperparameters(**model_kwarg)
# set the trainer
model.trainer = trainer
self.__log.info(
f'\n model.trainer.lr : {model.trainer.learning_rate}'
)
self.__log.info(
f'model.trainer.epochs : {model.trainer.epochs}\n')
# train the model
predictor = model.train(train_data)
# make predictions
forecast_it, ts_it = make_evaluation_predictions(
dataset=val_data, # test dataset
predictor=predictor, # predictor
num_eval_samples=
1, # number of sample paths we want for evaluation
)
# convert gluonts objects to lists
forecasts = list(forecast_it)
tss = list(ts_it)
# get prediction length
prediction_length = forecasts[0].mean.shape[0]
y_test_ = list(val_data)[0]['target']
y_preds_ = forecasts[0].mean
y_test_ = y_test_[-prediction_length:]
mae_ = mean_absolute_error(y_test_, y_preds_)
# Report the loss to comet
experiment.log_metric("loss", mae_)
experiment.end()
# initialize comet REST API to retrieve the best hyperparameters
comet_api = comet_ml.API(rest_api_key=rest_api_key)
project = comet_api.get(workspace=workspace,
project_name=optimizer.
experiment_kwargs['project_name'].lower())
# get the experiment ids
exp_ids = [x.id for x in project]
scores_df = pd.DataFrame(index=exp_ids, columns=['metric'])
# loop through the experiments within the comet project
for exp_id in exp_ids:
exp = comet_api.get(
f"{workspace}/{project_name.lower()}/{exp_id}")
scores_df.at[exp_id,
'metric'] = exp.get_metrics()[0]['metricValue']
scores_df.metric = scores_df.metric.map(float)
# get experiment_id of the best score
best_exp_id = scores_df.metric.idxmin()
# get the best experiment
exp = comet_api.get(
f"{workspace}/{project_name.lower()}/{best_exp_id}")
# get the best hyperparameters
best_params = {
x['name']: x['valueCurrent']
for x in exp.get_parameters_summary() if x['name'] != 'f'
}
# save best params in model_name-keyed dictionary for later use
self.best_params[model_name] = best_params
def main(alpha=0.5):
scaler, yscaler, X_train, X_test, y_train, y_test = process_data()
config = {
# We pick the Bayes algorithm:
'algorithm': 'bayes',
'name': 'fine tune LightGBM',
# Declare your hyperparameters in the Vizier-inspired format:
'parameters': {
'n_estimators': {
'type': 'integer',
'min': 50,
'max': 5000
},
'max_depth': {
'type': 'integer',
'min': 10,
'max': 50
},
'num_leaves': {
'type': 'integer',
'min': 100,
'max': 500
},
'reg_alpha': {
'type': 'float',
'min': 0.00001,
'max': 0.2,
'scalingType': 'loguniform',
},
'reg_lambda': {
'type': 'float',
'min': 0.00001,
'max': 0.2,
'scalingType': 'loguniform',
},
'subsample': {
'type': 'float',
'min': 0.2,
'max': 1.0
},
'colsample_bytree': {
'type': 'float',
'min': 0.2,
'max': 1.0
},
'min_child_weight': {
'type': 'float',
'min': 0.001,
'max': 0.1,
'scalingType': 'loguniform',
},
},
# Declare what we will be optimizing, and how:
'spec': {
'metric': 'loss',
'objective': 'minimize'
},
}
# Next, create an optimizer, passing in the config:
# (You can leave out API_KEY if you already set it)
opt = Optimizer(config,
api_key=os.environ['COMET_API_KEY'],
project_name='color-ml')
for _, experiment in enumerate(opt.get_experiments()):
experiment.log_parameter('colorspace', 'rgb')
params = {
'n_estimators': experiment.get_parameter('n_estimators'),
'colsample_bytree': experiment.get_parameter('colsample_bytree'),
'num_leaves': experiment.get_parameter('num_leaves'),
'max_depth': experiment.get_parameter('max_depth'),
'reg_alpha': experiment.get_parameter('reg_alpha'),
'reg_lambda': experiment.get_parameter('reg_lambda'),
'subsample': experiment.get_parameter('subsample'),
'min_child_weight': experiment.get_parameter('min_child_weight'),
}
loss, std = fit(experiment, X_train, y_train, params, alpha)
experiment.log_metric('loss', loss)
experiment.log_metric('std', std)
print("Action space shape:", ac_space)
assert ob_space is not None
#%%
config = {
"algorithm": "bayes",
"parameters": {
"lr": {"type": "float", "min": np.log(1e-5), "max": np.log(1e-1), "scalingType": "loguniform" },
},
"spec": {
"metric": "Speed",
"objective": "maximize",
},
}
optimizer = Optimizer(config, api_key="OZwyhJHyqzPZgHEpDFL1zxhyI",
project_name="rl-in-wifi")
#%%
teacher = Teacher(env, 1, Preprocessor(False))
# actor_l = [64, 32, 16]
# critic_l = [64, 32, 16]
# lr_actor = 1e-5
# lr_critic = 4e-5
for experiment in optimizer.get_experiments():
lr = experiment.get_parameter("lr")
# config = Config(buffer_size=4*steps_per_ep*threads_no, batch_size=256, gamma=0.98, tau=1e-3, lr_actor=lr_actor, lr_critic=lr_critic, update_every=1)
# lr = 1e-7
config = Config(buffer_size=4*steps_per_ep*threads_no, batch_size=256, gamma=0.8, tau=1e-3, lr=lr, update_every=1)
from sigopt.examples import franke_function
from comet_ml import Optimizer
import sys
def fit(x, y):
return franke_function(x, y)
#opt=Optimizer(sys.argv[1],api_key="6krXLYdn4mMFKPsF8jwrFwXtu",workspace="returncode13",project_name="franke-function-opt-distributed-01")
opt = Optimizer(sys.argv[1],
workspace="returncode13",
project_name="franke-function-opt-distributed-02")
for experiment in opt.get_experiments():
val = fit(experiment.get_parameter('x'), experiment.get_parameter('y'))
experiment.log_metric("val", val)
#print(experiment.get_parameter('x'),experiment.get_parameter('y'))
def main():
""" Main tracking script """
args = parse_args()
print(args)
from comet_ml import Optimizer
# We only need to specify the algorithm and hyperparameters to use:
config = {
# We pick the Bayes algorithm:
"algorithm": "bayes",
# Declare your hyperparameters in the Vizier-inspired format:
"parameters": {
"alpha": {
"type": "float",
"min": 0.01,
"max": 0.3
},
"lr": {
"type": "float",
"min": 5e-5,
"max": 5e-2
},
"gamma": {
"type": "float",
"min": 0.75,
"max": 0.99
},
},
# Declare what we will be optimizing, and how:
"spec": {
"metric": "VC",
"objective": "maximize",
"seed": args.rng_seed,
},
}
# Next, create an optimizer, passing in the config:
opt = Optimizer(config, project_name=args.experiment)
for experiment in opt.get_experiments():
experiment.auto_metric_logging = False
experiment.workspace = 'TrackToLearn'
experiment.parse_args = False
experiment.disabled = not args.use_comet
gamma = experiment.get_parameter("gamma")
alpha = experiment.get_parameter("alpha")
lr = experiment.get_parameter("lr")
td3_experiment = SACTrackToLearnTraining(
# Dataset params
args.path,
args.experiment,
args.name,
args.dataset_file,
args.subject_id,
args.test_dataset_file,
args.test_subject_id,
args.reference_file,
args.ground_truth_folder,
# RL params
args.max_ep,
args.log_interval,
args.action_std,
args.valid_noise,
lr,
gamma,
alpha,
# TD3 params
args.training_batch_size,
# Env params
args.n_seeds_per_voxel,
args.max_angle,
args.min_length,
args.max_length,
args.step_size, # Step size (in mm)
args.tracking_batch_size,
args.n_signal,
args.n_dirs,
# Model params
args.n_latent_var,
args.hidden_layers,
args.add_neighborhood,
# Experiment params
args.use_gpu,
args.rng_seed,
experiment,
args.render,
args.run_tractometer,
args.load_teacher,
args.load_policy,
)
td3_experiment.run()
def optimise(self):
config = {
# We pick the Bayes algorithm:
"algorithm": "bayes",
# Declare your hyperparameters in the Vizier-inspired format:
"parameters": {
"n_estimators": {
"type": "integer",
"min": self.n_estimators["min"],
"max": self.n_estimators["max"],
"scalingType": "uniform"
},
"max_depth": {
"type": "integer",
"min": self.max_depth["min"],
"max": self.max_depth["max"],
"scalingType": "uniform"
},
"learning_rate": {
"type": "float",
"min": self.learning_rate["min"],
"max": self.learning_rate["max"],
"scalingType": "uniform"
},
"gamma": {
"type": "float",
"min": self.gamma["min"],
"max": self.gamma["max"],
"scalingType": "uniform"
},
"subsample": {
"type": "float",
"min": self.subsample["min"],
"max": self.subsample["max"],
"scalingType": "uniform"
},
"min_child_weight": {
"type": "float",
"min": self.min_child_weight["min"],
"max": self.min_child_weight["max"],
"scalingType": "uniform"
},
"reg_alpha": {
"type": "float",
"min": self.alpha["min"],
"max": self.alpha["max"],
"scalingType": "uniform"
},
},
# Declare what we will be optimizing, and how:
"spec": {
"metric": "ROC",
"objective": "maximize",
},
}
opt = Optimizer(config,
api_key=self.comet_api_key,
project_name=self.comet_project_name,
auto_metric_logging=True)
for experiment in opt.get_experiments():
self.model.learning_rate = experiment.get_parameter(
"learning_rate")
self.model.n_estimators = experiment.get_parameter("n_estimators")
self.model.subsample = experiment.get_parameter("subsample")
self.model.max_depth = experiment.get_parameter("max_depth")
self.model.gamma = experiment.get_parameter("gamma")
self.model.reg_alpha = experiment.get_parameter("reg_alpha")
self.model.min_child_weight = experiment.get_parameter(
"min_child_weight")
self.train()
self.test()
auc, binary_accuracy = self.evaluate()
experiment.log_metric("ROC", auc)
experiment.log_metric("Binary_Accuracy", binary_accuracy)
def optimize(parsed, subcommand_args):
if parsed.OPTIMIZER is None:
parsed.OPTIMIZER = parsed.PYTHON_SCRIPT
parsed.PYTHON_SCRIPT = None
# Pass it on, in case it needs/wants it:
subcommand_args += [parsed.OPTIMIZER]
if parsed.trials is not None:
subcommand_args += ["--trials", str(parsed.trials)]
# Is the COMET_API_KEY available?
config = get_config()
api_key = config["comet.api_key"]
if api_key is None:
raise Exception(
"""Please set your API key: see https://www.comet.ml/docs/python-sdk/advanced/#python-configuration"""
)
if not (os.path.isfile(parsed.OPTIMIZER) or len(parsed.OPTIMIZER) == 32):
raise Exception("Optimizer should be either file or id: '%s'" %
parsed.OPTIMIZER)
# Create a new Optimizer, or use existing one:
if parsed.PYTHON_SCRIPT is None:
# Don't echo URL if PYTHON_SCRIPT isn't listed:
opt = Optimizer(parsed.OPTIMIZER, trials=parsed.trials, verbose=0)
else:
if not os.path.isfile(parsed.PYTHON_SCRIPT):
raise Exception("Python script file '%s' not found" %
parsed.PYTHON_SCRIPT)
opt = Optimizer(parsed.OPTIMIZER, trials=parsed.trials)
if parsed.dump is not None:
with open(parsed.dump, "w") as fp:
fp.write(str(opt.status()))
if parsed.PYTHON_SCRIPT is None:
# Just print the optimizer_id
print(opt.id)
# And exit
sys.exit(0)
environ = os.environ.copy()
environ["COMET_OPTIMIZER_ID"] = opt.id
COMET_EXECUTABLE = parsed.executable or sys.executable
bootstrap_dir = os.path.dirname(comet_ml.bootstrap.__file__)
# Prepend the bootstrap dir to a potentially existing PYTHON PATH, prepend
# so we are sure that we are the first one to be executed and we cannot be
# sure that other sitecustomize.py files would call us
if "PYTHONPATH" in environ:
if bootstrap_dir not in environ["PYTHONPATH"]:
environ["PYTHONPATH"] = "%s:%s" % (bootstrap_dir,
environ["PYTHONPATH"])
else:
environ["PYTHONPATH"] = bootstrap_dir
command_line = [COMET_EXECUTABLE, parsed.PYTHON_SCRIPT] + subcommand_args
subprocesses = [] # type: List[subprocess.Popen]
for j in range(parsed.parallel):
environ["COMET_OPTIMIZER_PROCESS_ID"] = str(j)
sub = subprocess.Popen(command_line, env=environ)
subprocesses.append(sub)
exit_code = 0
try:
for sub in subprocesses:
sub.wait()
if sub.returncode != 0:
exit_code = 1
except KeyboardInterrupt:
# Ask nicely for subprocesses to exit
for sub in subprocesses:
# TODO: Check behavior on Windows
sub.send_signal(signal.SIGINT)
# Check that all subprocesses exit cleanly
i = 0
while i < 60:
all_dead = True
for sub in subprocesses:
sub.poll()
alive = sub.returncode is None
all_dead = all_dead and not alive
if all_dead:
break
i += 1
time.sleep(1)
# Timeout, hard-kill all the remaining subprocess
if i >= 60:
for sub in subprocesses:
sub.poll()
if sub.returncode is None:
sub.kill()
print()
results = opt.status()
for key in ["algorithm", "status"]:
print(" ", "%s:" % key, results[key])
if isinstance(results["endTime"], float) and isinstance(
results["startTime"], float):
print(
" ",
"time:",
(results["endTime"] - results["startTime"]) / 1000,
"seconds",
)
sys.exit(exit_code)
"type": "discrete",
'values': [32, 64, 128, 256, 512]
},
"num_epochs": 1,
"learning_rate": 0.001
},
# Declare what we will be optimizing, and how:
"spec": {
"metric": "train_loss",
"objective": "minimize",
# "batch_size": 100,
},
}
optimizer = Optimizer(config, project_name="MNIST")
# hyper_params = {
# "sequence_length": 28,
# "input_size": 28,
# "hidden_size": 128,
# "num_layers": 2,
# "num_classes": 10,
# "batch_size": 100,
# "num_epochs": 10,
# "learning_rate": 0.01
# }
class Net(nn.Module):
def __init__(self):
def main():
opt = Optimizer("0FyTyofhjonvvEMrssqMng6pC")
experiment = Experiment(api_key="0FyTyofhjonvvEMrssqMng6pC",
project_name="route_experiment_horizontal",
workspace="pingakshya2008")
dataframe = pd.read_csv("C:/ddr_read/ISPD4/A3.csv")
#dataframe = pd.read_csv('C:/ddr_read/ISPD4/ispd2_ispd4.csv')
print(dataframe.head(5))
array = dataframe.values
# separate array into input and output components
X = array[:, 0:9]
Y = array[:, 10]
dataframe_test = pd.read_csv(
"C:/ddr_read/ISPD2/ispd2_final_horizontal.csv")
print(dataframe_test.head(5))
array_test = dataframe_test.values
# separate array into input and output components
X_test_test = array_test[:, 0:10]
Y_test_test = array_test[:, 11]
#Y = dataframe[' Hori route cong (%)']
print("----------------------xxxx-------------------")
print(X)
print("----------------------yyyy-------------------")
print(Y)
#scaler = MinMaxScaler(feature_range=(0, 1))
#rescaledX = scaler.fit_transform(X)
# summarize transformed data
scaler = MinMaxScaler(feature_range=(-1, 1))
rescaledX = scaler.fit_transform(X)
rescaledX_test = scaler.fit_transform(X_test_test)
numpy.set_printoptions(precision=4)
#print(rescaledX[3:9,:])
validation_size = 0.30
seed = 10
X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(
rescaledX, Y, test_size=validation_size, random_state=seed)
print("--- x train-------------")
print(X_train)
'''
# pcs for MLP
pcs_content = """hidden_layer_sizes integer [1000,2000] [1500]
solver categorical {sgd,adam,lbfgs} [adam]
activation categorical {identity,logistic,tanh,relu} [relu]
learning_rate categorical {constant,invscaling,adaptive} [constant]
"""
'''
i = 0
###pcs for random forest
pcs_content = """n_estimators integer [10,100] [11]
min_samples_split integer [2,20] [3]
min_samples_leaf real [0,0.499] [0.1]
max_features categorical {auto,sqrt,log2,None} [auto]
max_leaf_nodes integer [50,150] [100]
bootstrap categorical {True,False} [True]
"""
'''
### pcs for Linear Regression
pcs_content="""fit_intercept categorical {True,False} [True]
normalize categorical {True,False} [False]
copy_X categorical {True,False} [True]
"""
'''
opt.set_params(pcs_content)
while True:
i = i + 1
try:
sug = opt.get_suggestion()
except NoMoreSuggestionsAvailable:
break
print("SUG", sug, sug.__dict__)
#if i==700 :
# break
'''
##/ ** ** ** ** ** ** ** ** estimators for Linear Regression **** ** ** ** ** ** ** ** * /
fi=sug["fit_intercept"]
no= normalize=["normalize"]
cx=sug["copy_X"]
print("fit_intercept= ",repr(fi),"normalize= ",repr(no),"copy_X= ",repr(cx))
clf= LinearRegression(fit_intercept=sug["fit_intercept"], normalize=sug["normalize"], copy_X=sug["copy_X"])
'''
'''
# /**************** estimators for MLP *******************/
flu = sug["hidden_layer_sizes"]
sol=sug["solver"]
print("FLU", repr(flu))
print("sol", repr(sol))
clf = MLPRegressor(hidden_layer_sizes=sug["hidden_layer_sizes"], solver=sug["solver"],activation=sug["activation"],
alpha=0.0001, batch_size='auto', learning_rate=sug["learning_rate"], learning_rate_init=0.001,
power_t=0.05, max_iter=200, shuffle=True, random_state=10, tol=0.0001, verbose=False,
warm_start=False, momentum=0.9, nesterovs_momentum=True, early_stopping=False,
validation_fraction=0.05, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
'''
# /**************** estimators for Random Forrest Rgressor *******************/
ne = sug["n_estimators"]
ms = sug["min_samples_split"]
ml = sug["min_samples_leaf"]
mln = sug["max_leaf_nodes"]
bs = sug["bootstrap"]
oob = "false"
print("estimator= ", repr(ne), "mean sample split= ",
repr(ms), "min sample leaf= ", repr(ml), "max leaf nodes= ",
repr(mln), "bootstrap= ", repr(bs), "oob=", repr(oob), "i= ", i)
clf = sklearn.ensemble.RandomForestRegressor(
n_estimators=sug["n_estimators"],
criterion='mse',
max_depth=10,
min_samples_split=sug["min_samples_split"],
min_samples_leaf=sug["min_samples_leaf"],
max_features='auto',
max_leaf_nodes=sug["max_leaf_nodes"],
bootstrap=sug["bootstrap"],
oob_score=False,
n_jobs=1,
random_state=10,
verbose=0)
'''
activation='relu', solver='adam',
alpha=0.0001, batch_size='auto', learning_rate='constant', learning_rate_init=0.001,
power_t=0.05, max_iter=200, shuffle=True, random_state=10, tol=0.0001, verbose=False,
warm_start=False, momentum=0.9, nesterovs_momentum=True, early_stopping=False,
validation_fraction=0.05, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
'''
clf.fit(X_train, Y_train)
y_predict = clf.predict(X_validation)
print('R2 score validation %.5f' % r2_score(Y_validation, y_predict))
score = r2_score(Y_validation, y_predict)
sug.report_score("accuracy", score)
"type": "integer",
"min": 1,
"max": 3
},
"M": {
"type": "integer",
"min": 5,
"max": 30
},
},
"spec": {
"metric": "mAP",
"objective": "minimize",
},
}
opt = Optimizer(config, api_key=API_KEY, project_name="MSC-Thesis-PJ")
# In[ ]:
for experiment in opt.get_experiments():
N = experiment.get_parameter('N')
M = experiment.get_parameter('M')
train_generator = create_train_generator(N, M)
model = create_model()
history = model.fit_generator(generator=train_generator,
steps_per_epoch=10000,
epochs=5,
verbose=1)
test_results = test_model(model, create_test_generator(), 0.3)
experiment.log_metric("mAP", -test_results[2][0])
K.clear_session()
