def test_elapsed_real_time():
# Speed drop the first 5 observations
timer = ElapsedRealTime()
task = TaskMock(epochs=12,
steps=12,
callback=stop_after(None, None, sleep_time=0.01))
task.metrics.append(timer)
task.fit()
assert timer.value()['elapsed_time'] - 12 * 12 * 0.01 < 0.1
def show_progress():
epochs = 4
steps = 12
# Speed drop the first 5 observations
speed = Speed()
progress_default = ProgressView(speed)
progress_epoch_guess = ProgressView(speed, max_epochs=epochs)
progress_epoch = ProgressView(speed, max_epochs=epochs, max_steps=steps)
progress_steps = ProgressView(speed, max_steps=epochs * steps)
progress_default.print_fun = named_print('default')
progress_epoch_guess.print_fun = named_print(' guess')
progress_epoch.print_fun = named_print(' epoch')
progress_steps.print_fun = named_print(' step')
task = TaskMock(callback=stop_after(None, None, sleep_time=1),
epochs=epochs,
steps=steps)
task.metrics.append(speed)
task.metrics.append(ElapsedRealTime())
task.metrics.append(progress_default)
task.metrics.append(progress_epoch_guess)
task.metrics.append(progress_epoch)
task.metrics.append(progress_steps)
task.fit()
def __init__(self,
detector,
optimizer,
lr_scheduler,
dataloader,
criterion=None,
device=None,
storage=None):
super(ObjectDetection, self).__init__(device=device)
self._first_epoch = 0
self.current_epoch = 0
self.detector = detector
self.optimizer = optimizer
self.lr_scheduler = lr_scheduler
self.dataloader = dataloader
self.criterion = criterion
self.storage = storage
self.metrics.append(ElapsedRealTime().every(batch=1))
self.metrics.append(SampleCount().every(batch=1, epoch=1))
speed = Speed()
self.metrics.append(speed)
self.metrics.append(ProgressView(speed))
self.metrics.append(OnlineLoss())
if storage:
self.metrics.append(CheckPointer(storage=storage))
def __init__(self, classifier, optimizer, lr_scheduler, dataloader, criterion=None, device=None,
storage=None, preprocessor=None, metrics=None):
super(Classification, self).__init__(device=device)
criterion = select(criterion, CrossEntropyLoss())
self._first_epoch = 0
self.current_epoch = 0
self.classifier = classifier
self.optimizer = optimizer
self.lr_scheduler = lr_scheduler
self.dataloader = dataloader
self.criterion = criterion
self.preprocessor = Preprocessor()
# ------------------------------------------------------------------
self.metrics.append(ElapsedRealTime().every(batch=1))
self.metrics.append(SampleCount().every(batch=1, epoch=1))
self.metrics.append(OnlineTrainAccuracy())
self.metrics.append(Speed())
# All metrics must be before ProgressView and CheckPointer
if metrics:
for metric in metrics:
self.metrics.append(metric)
self.metrics.append(ProgressView(self.metrics.get('Speed')))
if storage:
self.metrics.append(CheckPointer(storage=storage))
# ------------------------------------------------------------------
if preprocessor is not None:
self.preprocessor = preprocessor
self.hyper_parameters = {}
def __init__(self,
dataset,
oracle,
model,
optimizer,
device,
criterion=SharpeRatioCriterion()):
super(Finance, self).__init__(device=device)
self.dataset = dataset
self.oracle = oracle
self.criterion = criterion
self.model = model
self.optimizer = optimizer
self.metrics.append(ElapsedRealTime().every(batch=1))
self.metrics.append(SampleCount().every(batch=1, epoch=1))
speed = Speed()
self.metrics.append(speed)
self.metrics.append(ProgressView(speed))
self.metrics.append(OnlineLoss())
self.metrics.append(NamedMetric(name='mean_returns'))
self.metrics.append(NamedMetric(name='std_returns'))
self.current_epoch = 0
self.hyper_parameters = {}
def main(**kwargs):
show_dict(kwargs)
args = Namespace(**kwargs)
set_verbose_level(args.verbose)
device = fetch_device()
experiment_name = args.experiment_name.format(**kwargs)
# save partial results here
state_storage = StateStorage(
folder=option('state.storage', '/tmp/olympus/classification'))
def main_task():
task = classification_baseline(device=device,
storage=state_storage,
**kwargs)
if args.uri is not None:
logger = metric_logger(args.uri, args.database, experiment_name)
task.metrics.append(logger)
return task
space = main_task().get_space()
# If space is not empty we search the best hyper parameters
params = {}
if space:
show_dict(space)
hpo = HPOptimizer('hyperband',
space=space,
fidelity=Fidelity(args.min_epochs,
args.epochs).to_dict())
hpo_task = HPO(hpo, main_task)
hpo_task.metrics.append(ElapsedRealTime())
trial = hpo_task.fit(objective='validation_accuracy')
print(f'HPO is done, objective: {trial.objective}')
params = trial.params
else:
print('No hyper parameter missing, running the experiment...')
# ------
# Run the experiment with the best hyper parameters
# -------------------------------------------------
if params is not None:
# Train using train + valid for the final result
final_task = classification_baseline(device=device,
**kwargs,
hpo_done=True)
final_task.init(**params)
final_task.fit(epochs=args.epochs)
print('=' * 40)
print('Final Trial Results')
show_dict(flatten(params))
final_task.report(pprint=True, print_fun=print)
print('=' * 40)
def __init__(self, model, metrics, name=None):
super(SklearnTask, self).__init__()
self.model = model
# Measure the time spent training
self.metrics.name = name
self.metrics.append(ElapsedRealTime().every(batch=1))
self.metrics.append(SampleCount().every(batch=1))
for metric in metrics:
self.metrics.append(metric)
def __init__(self, models, create_subtask_metrics):
super(SklearnEnsembleTask, self).__init__()
self.name = ''
self.models = models
self.tasks = OrderedDict()
for name, model in sorted(models.items()):
self.tasks[name] = SklearnTask(model, create_subtask_metrics(name), name=name)
# Measure the time spent training
self.metrics.append(ElapsedRealTime().every(batch=1))
self.metrics.append(SampleCount().every(batch=1))
self.metrics.append(EnsembleMetric(self))
def __init__(self,
model: AbstractActorCritic,
dataloader,
optimizer,
lr_scheduler,
device,
ppo_epoch=5,
ppo_batch_size=32,
ppo_clip_param=10,
ppo_max_grad_norm=1000,
criterion=None,
storage=None,
logger=None):
super(PPO, self).__init__(device=device)
if criterion is None:
criterion = lambda x: x.sum()
self.actor_critic = model
self.lr_scheduler = lr_scheduler
self.optimizer: Optimizer = optimizer
self.criterion: Module = criterion
self.gamma: float = 0.99
self.eps = np.finfo(np.float32).eps.item()
self.action_sampler: Callable[[], Distribution] = Categorical
self.tensor_shape = None
self.frame_count: int = 0
self.dataloader = dataloader
self.storage = storage
self._first_epoch = 0
self.current_epoch = 0
self.ppo_epoch = ppo_epoch
self.ppo_batch_size = ppo_batch_size
self.ppo_clip_param = ppo_clip_param
self.ppo_max_grad_norm = ppo_max_grad_norm
self.metrics.append(NamedMetric(name='loss'))
self.metrics.append(ElapsedRealTime())
self.metrics.append(Speed())
self.metrics.append(ProgressView(self.metrics.get('Speed')))
if storage:
self.metrics.append(CheckPointer(storage=storage))
self.hyper_parameters = {}
self.batch_size = None