def test_wrong_construction_2(self, dataloader_dummy):
"""Wrong keyword arguments."""
with pytest.raises(TypeError):
Run(DummyNet(), MeanReturns(), dataloader_dummy, metrics='this_is_fake')
with pytest.raises(TypeError):
Run(DummyNet(), MeanReturns(), dataloader_dummy, metrics={'a': 'this_is_fake'})
with pytest.raises(ValueError):
Run(DummyNet(), MeanReturns(), dataloader_dummy, metrics={'loss': MeanReturns()})
with pytest.raises(TypeError):
Run(DummyNet(), MeanReturns(), dataloader_dummy, val_dataloaders='this_is_fake')
with pytest.raises(TypeError):
Run(DummyNet(), MeanReturns(), dataloader_dummy, val_dataloaders={'val': 'this_is_fake'})
with pytest.raises(TypeError):
Run(DummyNet(), MeanReturns(), dataloader_dummy, benchmarks='this_is_fake')
with pytest.raises(TypeError):
Run(DummyNet(), MeanReturns(), dataloader_dummy, benchmarks={'uniform': 'this_is_fake'})
with pytest.raises(ValueError):
Run(DummyNet(), MeanReturns(), dataloader_dummy, benchmarks={'main': OneOverN()})
def train_model(network,
train_dataloader,
val_dataloaders,
optimizer,
callbacks,
epochs=20,
device="cpu",
loss_="sharpe"):
if loss_ == "sharpe":
loss = SharpeRatio(returns_channel=0)
else:
loss = MaximumDrawdown(returns_channel=0)
benchmarks = {"1overN": OneOverN()}
metrics = {
"drawdown": MaximumDrawdown(returns_channel=0),
"sharpe": SharpeRatio(returns_channel=0)
}
run = Run(
network,
loss,
train_dataloader,
val_dataloaders=val_dataloaders,
metrics=metrics,
# benchmarks=benchmarks,
device=torch.device(device),
optimizer=optimizer,
callbacks=callbacks,
)
history = run.launch(n_epochs=epochs)
return run
def run_dummy(dataloader_dummy, network_dummy, Xy_dummy):
""""""
X_batch, y_batch, timestamps, asset_names = Xy_dummy
device = X_batch.device
dtype = X_batch.dtype
return Run(network_dummy,
MeanReturns(),
dataloader_dummy,
val_dataloaders={'val': dataloader_dummy},
benchmarks={'bm': OneOverN()},
device=device,
dtype=dtype)
def test_basic(self, Xy_dummy):
X_dummy, _, _, _ = Xy_dummy
n_samples, n_channels, lookback, n_assets = X_dummy.shape
dtype = X_dummy.dtype
device = X_dummy.device
bm = OneOverN()
weights = bm(X_dummy)
assert isinstance(weights, torch.Tensor)
assert weights.dtype == dtype
assert weights.device == device
assert torch.allclose(
weights.sum(dim=1),
torch.ones(n_samples).to(dtype=dtype, device=device))
assert len(torch.unique(weights)) == 1
assert isinstance(bm.hparams, dict) and not bm.hparams
def test_attributes_after_construction(self, dataloader_dummy, additional_kwargs):
network = DummyNet()
loss = MeanReturns()
kwargs = {}
if additional_kwargs:
kwargs.update({'metrics': {'std': StandardDeviation()},
'val_dataloaders': {'val': dataloader_dummy},
'benchmarks': {'whatever': OneOverN()}})
run = Run(network, loss, dataloader_dummy, **kwargs)
assert network is run.network
assert loss is run.loss
assert dataloader_dummy is run.train_dataloader
assert isinstance(run.metrics, dict)
assert isinstance(run.val_dataloaders, dict)
'train':
dataloader,
'val':
RigidDataLoader(dataset,
indices=list(range(5020, 9800)),
batch_size=batch_size,
lookback=lookback)
}
run = Run(network,
100 * MeanReturns(),
dataloader,
val_dataloaders=val_dataloaders,
metrics={'sqweights': SquaredWeights()},
benchmarks={
'1overN': OneOverN(),
'VAR': VARTrue(process),
'Random': Random(),
'InverseVol': InverseVolatility()
},
optimizer=torch.optim.Adam(network.parameters(), amsgrad=True),
callbacks=[EarlyStoppingCallback('val', 'loss')])
history = run.launch(40)
fig, ax = plt.subplots(1, 1)
ax.set_title('Validation loss')
per_epoch_results = history.metrics.groupby(
['dataloader', 'metric', 'model', 'epoch'])['value'].mean()['val']['loss']
our = per_epoch_results['network']
print(per_epoch_results.mean()) # mean loss per epoch
# %%
per_epoch_results.mean()['test']['loss']['network'].plot()
# %%
# To get more insight into what our network predicts we can use the :code:`deepdow.visualize` module.
# Before we even start further evaluations, let us make sure the network is in eval model.
network = network.eval()
# %%
# To put the performance of our network in context, we also utilize benchmarks. :code:`deepdow`
# offers multiple benchmarks already. Additionally, one can provide custom simple benchmarks or
# some pre-trained networks.
benchmarks = {
'1overN': OneOverN(), # each asset has weight 1 / n_assets
'random': Random(), # random allocation that is however close 1OverN
'network': network
}
# %%
# During training, the only mandatory metric/loss was the loss criterion that we tried to minimize.
# Naturally, one might be interested in many other metrics to evaluate the performance. See below
# an example.
metrics = {
'MaxDD': MaximumDrawdown(),
'Sharpe': SharpeRatio(),
'MeanReturn': MeanReturns()
}