def get(self, configuration):
experiment = Experiment()
experiment.model_trainer = self.model_trainer_factory.get(
configuration)
experiment.model_tester = self.model_tester_factory.get(configuration)
experiment.model = self.model_factory.get(configuration)
return experiment
def __init__(self, logger_obj, n, i, seed=5):
folds = n_fold_split(Amigos.SUBJECTS_IDS, n, seed=seed)
self.test_ids = folds[i]["test"]
self.val_ids = folds[i]["val"]
self.train_ids = folds[i]["train"]
Experiment.__init__(self, "Amigos", logger_obj, SIGNALS_LEN, dataset_name_suffix=f"_{n}fold_{i:02d}")
def test_save(self, tmp_path):
dataset_path = Path('ml/input/test/sample.csv')
o = ProjectModel()
artifacts_handler = ExperimentArtifacts(
run_tag='test',
model_name=o.model_id,
base_path=tmp_path,
)
e = Experiment(
run_tag='test',
model=o,
input_dir=dataset_path,
artifacts_handler=artifacts_handler,
)
e.run()
assert o.model_path.is_file()
def test_load(self, tmp_path):
dataset_path = Path('ml/input/test/sample.csv')
o = ProjectModel()
artifacts_handler = ExperimentArtifacts(
run_tag='test',
model_name=o.model_id,
base_path=tmp_path,
)
e = Experiment(
run_tag='test',
model=o,
input_dir=dataset_path,
artifacts_handler=artifacts_handler,
)
e.run()
o2 = ProjectModel()
o2.load(tmp_path)
model = o.model
model_ = o2.model
assert type(model_) == type(
model) # must return the same type of object
assert model_ is not model # object identity MUST be different
def test_predict(self, tmp_path):
dataset_path = Path('ml/input/test/sample.csv')
o = ProjectModel()
artifacts_handler = ExperimentArtifacts(
run_tag='test',
model_name=o.model_id,
base_path=tmp_path,
)
e = Experiment(
run_tag='test',
model=o,
input_dir=dataset_path,
artifacts_handler=artifacts_handler,
)
e.run()
p = o.predict(
ids=["1"],
X=[
"406,-2.3122265423263,1.95199201064158,-1.60985073229769,3.9979055875468,-0.522187864667764,-1.42654531920595,-2.53738730624579,1.39165724829804,-2.77008927719433,-2.77227214465915,3.20203320709635,-2.89990738849473,-0.595221881324605,-4.28925378244217,0.389724120274487,-1.14074717980657,-2.83005567450437,-0.0168224681808257,0.416955705037907,0.126910559061474,0.517232370861764,-0.0350493686052974,-0.465211076182388,0.320198198514526,0.0445191674731724,0.177839798284401,0.261145002567677,-0.143275874698919,0,1"
.split(",")[:-1]
], # noqa: E501
)
assert "1" in p
assert p["1"] in [0.0, 1.0]
from skopt.space import Categorical
from conf import conf
from data import registry
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/mv_nonlinear')
conf.num_workers = 4
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
exp.data_loader = registry.mv_nonlinear()
exp.model_factory = TfTrainEvalModelFactory(
Model(name="RNADE_deep_normal"))
exp.hyper_param_search = GridSearch([
Categorical([32, 64, 128], name='km'),
Categorical([64, 128, 512], name='sh'),
Categorical([1, 2, 3, 4, 5], name='nh'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
Categorical(['AdamOptimizer'], name='opt'),
Categorical([1e-4, 1e-3, 1e-2], name='opt_lr'),
])
from skopt.space import Categorical
from conf import conf
from data import registry
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/sin_t')
conf.num_workers = 4
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
exp.data_loader = registry.sin_t_noise()
exp.model_factory = TfTrainEvalModelFactory(Model(name="RNADE_laplace"))
exp.hyper_param_search = GridSearch([
Categorical([1, 16, 32, 64, 128], name='km'),
Categorical([1, 16, 32, 64, 128], name='sh'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
Categorical(['AdamOptimizer'], name='opt'),
Categorical([1e-4, 1e-3, 1e-2], name='opt_lr'),
])
exp.early_stopping = EarlyStop(monitor_every_epoch=1, patience=[30])
for i in range(len(pop_list)):
fitness *= self.cal_pop_fitness(hmm, pop_list[i],
observation_list[i])
return fitness
def roulette_wheel_selection(self, population, fitness):
"""Perform roulette wheel selection given the current population"""
next_population = np.zeros(population.shape, dtype=int)
normalized_fitness = fitness / sum(fitness)
for i in range(0, self.population_size):
next_population[i, :] = population[
np.random.choice(population.shape[0], p=normalized_fitness), :]
return next_population
if __name__ == "__main__":
experiment = Experiment(trans_file="test_hmm_transition_matrix.csv",
emis_file="hmm_observations_emission.csv",
n_train_samples=2,
n_test_samples=2,
sample_length=20,
seed=1)
experiment.initialize()
possible_states = [
list(x) for x in list(product(np.arange(4), np.arange(4)))
]
filter = ParticleFilter(experiment)
print(experiment.y_test)
filter.run()
print(filter.y_pred)
from experiment.experiment import Experiment
import numpy as np
from itertools import product
import os
from experiment.result_analysis import ResultAnalysis
"""Main file: running the experiments with predetermined settings"""
if __name__ == "__main__":
DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources'))
# listing the possible separable states given the hmm
possible_states = [list(x) for x in list(product(np.arange(4), np.arange(4)))]
# listing the algorithms investigated in the experiment
gfilter4 = GeneticFilter("AGF4", possible_states, genetic_operation_resolution=4)
gfilter5 = GeneticFilter("AGF5", possible_states, genetic_operation_resolution=5)
gfilter6 = GeneticFilter("AGF6", possible_states, genetic_operation_resolution=6)
pfilter = ParticleFilter("APF")
viterbi = Viterbi("VA")
algorithms = [gfilter4, gfilter5, gfilter6, pfilter, viterbi]
# creating the experiment
experiment = Experiment(algorithms=algorithms,
trans_file=os.path.join(DATA_DIR, 'test_2_hmm_transition_matrix.csv'),
emis_file=os.path.join(DATA_DIR, 'hmm_observations_emission.csv'),
n_train_samples=10, n_test_samples=5000, sample_length=100, seed=1)
# initialize and run the experiment
experiment.initialize()
experiment.run()
# analyzing the results obtained in the experiment
result = ResultAnalysis(experiment, algorithms)
result.run()
from skopt.space import Categorical
from conf import conf
from data import registry
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/inv_sin_t')
conf.num_workers = 4
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
exp.data_loader = registry.inv_sin_t()
exp.model_factory = TfTrainEvalModelFactory(
Model(name="MONDE_copula_const_cov"))
exp.hyper_param_search = GridSearch([
Categorical([32, 64, 128], name='hxy_sh'),
Categorical([1, 2, 3], name='hxy_nh'),
Categorical([32, 64, 128], name='x_sh'),
Categorical([1, 2, 3], name='x_nh'),
Categorical([16, 32], name='hxy_x'),
Categorical([0.05, 0.01], name='clr'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
from skopt.space import Categorical
from conf import conf
from data import registry
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/inv_sin_normal')
conf.num_workers = 4
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
exp.data_loader = registry.inv_sin_normal()
exp.model_factory = TfTrainEvalModelFactory(Model(name="MAF"))
exp.hyper_param_search = GridSearch([
Categorical([1, 2, 3, 4, 5], name='nb'),
Categorical([32, 128, 256], name='sh'),
Categorical([1, 2, 3], name='nh'),
Categorical([16, 32, 64], name='shc'),
Categorical([True, False], name='bn'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
Categorical(['AdamOptimizer'], name='opt'),
Categorical([1e-4, 1e-3, 1e-2], name='opt_lr'),
for t, b in enumerate(beaches):
print b
task = tasks[t]
ma = MultiAgent()
print 'Creating agents...'
for i in range(noAgents):
learner = QLearner(features, task.env.noActions, i)
agent = Agent(learner,
num_features=features,
num_actions=task.env.noActions,
num_agents=noAgents,
index=i)
ma.addAgent(agent)
exp = Experiment(task, ma)
print 'Running experiment...'
distribution = exp.doTrials(number=trials)
results = task.globalRewards
saveToCSV(results, b, '00_' + comment + typeR, 'RND-SGM')
sigma = np.array([np.std(r) for r in results])
mean = np.array([np.mean(r) for r in results])
win = 500
err = [sigma[i] for i in range(0, len(mean), win)]
avg = [mean[i] for i in range(0, len(mean), win)]
l, = plt.plot(mean, color=colors[t], label=b, alpha=0.8)
from skopt.space import Categorical
from conf import conf
from data import registry
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/mv_nonlinear')
conf.num_workers = 4
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
exp.data_loader = registry.mv_nonlinear()
exp.model_factory = TfTrainEvalModelFactory(Model(name="PumondePFor"))
exp.hyper_param_search = GridSearch([
Categorical([64, 128, 256], name='xs'),
Categorical([1, 2, 3], name='xn'),
Categorical([64, 128, 256], name='hxys'),
Categorical([1, 2, 3], name='hxyn'),
Categorical([0, 16], name='hxyxs'),
Categorical([64, 128, 256], name='xycs'),
Categorical([1, 2, 3], name='xycn'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
controller.initialize(0.)
learner = Q(
epsilon=0.5
) # comes by default with explorer = EpsilonGreedyExplorer(epsilon = 0.3, decay = 0.9999)
agent = LearningAgent(controller, learner)
print('New Q agent created')
environment = SupplyChainEnv(normalized_prod_dataset[0:48],
normalized_price_dataset[0:48], discrete_battery,
discrete_vessel, discrete_elec_regime,
discrete_grid_price, discrete_prod,
discrete_buy_sell)
task = SupplyTaskMDP(environment)
experiment = Experiment(task, agent)
#Training
num_epochs = 1000
horizon = 48 # 24 interactions = horizon of 1 day
battery_states = []
h2_vessel_states = []
actions_electrolyzer = []
actions_grid = []
eval_rewards = []
for epoch in range(num_epochs):
actions_electrolyzer_day = []
actions_grid_day = []
battery_states_day = []
h2_vessel_states_day = []
from skopt.space import Categorical
from conf import conf
from data import registry
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/sin_normal')
conf.num_workers = 4
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
exp.data_loader = registry.sin_normal_noise()
exp.model_factory = TfTrainEvalModelFactory(Model(name="MONDE_AR_MADE"))
exp.hyper_param_search = GridSearch([
Categorical(['sigm'], name='tr'),
Categorical([32, 64, 128], name='sh'),
Categorical([1, 2, 3], name='nh'),
Categorical([16], name='xs'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
Categorical(['AdamOptimizer'], name='opt'),
Categorical([1e-4, 1e-3, 1e-2], name='opt_lr'),
])
from skopt.space import Categorical
from conf import conf
from data.registry import fx
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.conf import tf_conf
from models.tensorflow.pumonde2 import Pumonde2
from models.tensorflow.tf_simple_train_eval import TfSimpleTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('classification/fx_3')
conf.max_num_epochs = -1
conf.num_workers = 6
conf.visible_device_list = [0, 1]
conf.shuffle_train_data = True
conf.precision = "32"
conf.eval_batch_size = 10000
tf_conf.eval_throttle_secs = 0
tf_conf.save_summary_epochs = 1
tf_conf.save_checkpoints_epochs = 1
tf_conf.check_nans = True
tf_conf.start_eval_step = 1
tf_conf.per_process_gpu_memory_fraction = 0.2
symbols = ["AUDCAD", "AUDJPY", "AUDNZD", "EURCHF", "NZDCAD", "NZDJPY", "NZDUSD", "USDCHF", "USDJPY",
"EURUSD", "GBPUSD", "USDCAD"]
@staticmethod
def modify_date_time():
"""Utility method for experiment results folder saving"""
result = str(datetime.datetime.today().replace(microsecond=0))
for char in "-: ":
result = result.replace(char, "_")
return result
if __name__ == "__main__":
DATA_DIR = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'resources'))
experiment = Experiment(
trans_file=os.path.join(DATA_DIR, 'test_hmm_transition_matrix.csv'),
emis_file=os.path.join(DATA_DIR, 'hmm_observations_emission.csv'),
n_train_samples=2,
n_test_samples=2,
sample_length=20,
seed=1)
experiment.initialize()
possible_states = [
list(x) for x in list(product(np.arange(4), np.arange(4)))
]
filter = GeneticFilter(experiment, "a", possible_states)
filter.run()
viterbi = Viterbi(experiment, "v")
viterbi.run()
result = ResultAnalysis(experiment, [filter, viterbi])
result.run()
disaggregated_clairvoyant_rewards = [
optimal_disaggregated_super_arm_value
for _ in range(timesteps_context_generation)
]
############################################
## Define GPTS prior
############################################
GPTS_prior = lambda x: 3 * x
############################################
## Perform experiments
############################################
experiment = Experiment(original_environment, budget_discretization_steps,
daily_budget, GPTS_prior)
print("------ GPTS stationary ------")
(stationary_rewards, stationary_final_environment,
stationary_final_subcampaign_algos, regression_errors_max,
regression_errors_sum) = experiment.perform(timesteps_stationary)
print("------ GPTS context generation ------")
(context_generation_rewards, context_generation_final_environment,
context_generation_final_subcampaign_algos) = experiment.perform(
timesteps_context_generation, context_generation_rate)
############################################
## Plot results
############################################
from skopt.space import Categorical
from conf import conf
from data import registry
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/uci_large/hepmass')
conf.num_workers = 2
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 20000, '1': 20000}
exp.data_loader = registry.hepmass(x_slice=slice(0), y_slice=slice(None))
exp.model_factory = TfTrainEvalModelFactory(Model(name="MONDE_AR_BLOCK"))
exp.hyper_param_search = GridSearch([
Categorical([8, 10], name='nl'),
Categorical([60, 80], name='nb'),
Categorical(['tanh'], name='tr'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
Categorical(['AdamOptimizer'], name='opt'),
Categorical([1e-3], name='opt_lr'),
])
import logging
from experiment.experiment import Experiment
from ms_logging.ms_logger_configurator import MsLoggerConfigurator
from scenarios.ac_internal_scenario import AcInternalScenario
from scenarios.dc_current_pulse_scenario import DcCurrentPulseScenario
from scenarios.dc_linear_sweep_scenario import DcLinearSweepScenario
from scenarios.ocv_test_scenario import OcvTestScenario
from scenarios.ten_seconds_discharge import TenSecondsDischargeScenario
if __name__ == '__main__':
MsLoggerConfigurator().configure_logging(log_level=logging.INFO)
log = logging.getLogger("test")
recipes = [
OcvTestScenario,
AcInternalScenario,
DcCurrentPulseScenario,
DcLinearSweepScenario,
TenSecondsDischargeScenario
]
e = Experiment(recipes, 'dummy_experiment')
e.run()
skip_params = {'num_input_channels': config['maps_number'],
'num_channels_down': [8, 16, 24],
'num_channels_up': [8, 16, 24],
'num_channels_skip': [4, 4, 4]}
pregrid_params = {'num_input_channels': config['maps_number'],
'num_output_channels': 2,
'num_channels_down': [8, 16, 24],
'num_channels_up': [8, 16, 24],
'num_channels_skip': [4, 4, 4]}
config['net_params'] = [skip_params, pregrid_params]
net = spatial.Net(input_depth=config['maps_number'], pic_size=config['input_size'], skip_args_main=skip_params,
skip_args_grid=pregrid_params).type(dtype)
video = generateSyntheticData()
target = preprocessTarget(video, config["video_length"], config["output_size"])
loss = SpatialLoss()
spatial_maps_generator = SpatialMapsGenerator(config["maps_number"])
batch_generator = BatchGenerator(target, spatial_maps_generator, config[
"input_size"], config["input_size"])
parameters = list(net.parameters())
for var in spatial_maps_generator.spatial_variables.values():
parameters.append(var)
experiment = Experiment(config, parameters, batch_generator, net, loss)
experiment.run()
))
# ロガーに追加
logger.addHandler(stream_handler)
logger.info(f"Read Hyper Parameter file.")
hp = HP(name="hartmann6_test",
file_path="params/sample_param_hartmann6.yaml",
logger=logger)
pprint(hp.to_dict())
# check hp
logger.info(f"Check contents of Hyper Param.")
hp.check_params(Hartmann6Trainer.required_params)
# prepare and exec the experiment
logger.info(f"Exec Experiment.")
experiment = Experiment(hp, Hartmann6Trainer, logger=logger)
experiment.run()
best_result_x, best_result_y = experiment.get_best_result()
best_result_x = best_result_x.detach().numpy().tolist(
) if best_result_x is not None else "No variable exists"
best_result_y = best_result_y.detach().numpy().tolist()[0]
logger.info(
f'The best result of the experiment is (x:{hp.get_var_dict_from_tensor(best_result_x)}, y:{best_result_y}.'
)
# See PyCharm help at https://www.jetbrains.com/help/pycharm/
from skopt.space import Categorical
from conf import conf
from data import registry
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('classification/fx')
conf.num_workers = 4
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
symbols = [
"AUDCAD", "AUDJPY", "AUDNZD", "EURCHF", "NZDCAD", "NZDJPY", "NZDUSD",
"USDCHF", "USDJPY", "EURUSD", "GBPUSD", "USDCAD"
]
exp.data_loader = registry.fx(x_slice=slice(None, -3),
y_slice=slice(-3, None),
ar_terms=1,
start='2018-01-01',
end='2018-03-31',
symbols=symbols,
predicted_idx=None,
resample="1min")
import pytest
from experiment.experiment import Experiment
edit_distance_tests = [
pytest.param([], [], 0, id='trivial'),
pytest.param(['P', 'O', 'L', 'Y', 'N', 'O', 'M', 'I', 'A', 'L'],
['E', 'X', 'P', 'O', 'N', 'E', 'N', 'T', 'I', 'A', 'L'],
6,
id='correct numebr of edits'),
pytest.param(['P', 'O', 'L', 'Y', 'N', 'O', 'M', 'I', 'A', 'L'], [],
10,
id='tracks removing all elements'),
pytest.param([], ['H', 'E', 'L', 'L', 'O'],
5,
id='works for only adding characters'),
pytest.param(['H', 'E', 'L', 'L', 'O'], ['J', 'E', 'L', 'L', 'O'],
1,
id='works when only one replacement is needed'),
pytest.param(['H', 'E', 'L', 'L', 'O'], ['A', 'B', 'C', 'D', 'E'],
5,
id='works when all letters need to be replaced')
]
exp = Experiment()
@pytest.mark.parametrize('M, N, result_ed', edit_distance_tests)
def test_edit_distance(M, N, result_ed):
assert exp.edit_distance(M, N) == result_ed
from skopt.space import Categorical
from conf import conf
from data.data_utils import PercentileAnyGreaterLabelTransform
from data.registry import fx
from experiment.early_stop import EarlyStop
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.conf import tf_conf
from models.tensorflow.nn_classifier import NNClassifier
from models.tensorflow.tf_simple_train_eval import TfSimpleTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('classification/fx_3')
conf.max_num_epochs = -1
conf.num_workers = 2
conf.visible_device_list = [0, 1]
conf.shuffle_train_data = True
conf.precision = "32"
conf.eval_batch_size = 10000
tf_conf.eval_throttle_secs = 0
tf_conf.save_summary_epochs = 1
tf_conf.save_checkpoints_epochs = 1
tf_conf.check_nans = True
tf_conf.start_eval_step = 1
tf_conf.per_process_gpu_memory_fraction = 0.2
symbols = [
"AUDCAD", "AUDJPY", "AUDNZD", "EURCHF", "NZDCAD", "NZDJPY", "NZDUSD",
#!/usr/bin/env python3
import speech_recognition as sr
from pathlib import Path
import json
from os import path
from experiment.experiment import Experiment
from google_cloud import sample_long_running_recognize
exp = Experiment()
# use the audio file as the audio source
r = sr.Recognizer()
# recognize speech using Sphinx
try:
# print("Sphinx thinks you said " + r.recognize_sphinx(audio))
exp.run_experiment('CMUSphinx', r.recognize_sphinx, 'sound_bytes')
except sr.UnknownValueError:
print("Sphinx could not understand audio")
except sr.RequestError as e:
print("Sphinx error; {0}".format(e))
exp.run_experiment('GoogleCloud', sample_long_running_recognize, 'uri')