def get_labels_numba(price_series_np_array, t_events_np_array_epoch,
look_forward_window, min_sample_length, step):
t1_array_numba = np.zeros(
len(t_events_np_array_epoch)) # Array of label end times
t1_array_numba[:] = np.nan
t_values_array_numba = np.zeros(
len(t_events_np_array_epoch)) # Array of trend t-values
t_values_array_numba[:] = np.nan
for i in prange(len(t_events_np_array_epoch)):
subset_np_array = price_series_np_array[i:i + look_forward_window]
subset_np_array_epoch = t_events_np_array_epoch[i:i +
look_forward_window]
if len(subset_np_array) >= look_forward_window:
# Loop over possible look-ahead windows to get the one which yields maximum t values for b_1 regression coef
max_abs_t_value = -np.inf # Maximum abs t-value of b_1 coefficient among l values
max_t_value_index = None # Index with maximum t-value
max_t_value = None # Maximum t-value signed
for forward_window in range(min_sample_length,
len(subset_np_array), step):
y_subset = subset_np_array[:forward_window].reshape(
-1, 1) # y{t}:y_{t+l}
y_subset_np_array_epoch = subset_np_array_epoch[:
forward_window]
# Array of [1, 0], [1, 1], [1, 2], ... [1, l] # b_0, b_1 coefficients
X_subset = np.ones((y_subset.shape[0], 2))
X_subset[:, 1] = np.arange(y_subset.shape[0])
# Get regression coefficients estimates
# start = time.time()
b_mean_, b_std_ = get_betas(X_subset, y_subset)
# end = time.time()
# print("get_betas time")
# print(end - start)
# Check if l gives the maximum t-value among all values {0...L}
t_beta_1 = (b_mean_[1] / np.sqrt(b_std_[1, 1]))[0]
if abs(t_beta_1) > max_abs_t_value:
max_abs_t_value = abs(t_beta_1)
max_t_value = t_beta_1
max_t_value_index = forward_window
label_endtime_index = y_subset_np_array_epoch[max_t_value_index -
1]
# import pdb
# pdb.set_trace()
t1_array_numba[i] = label_endtime_index
t_values_array_numba[i] = max_t_value
else:
t1_array_numba[i] = np.nan
t_values_array_numba[i] = np.nan
return t1_array_numba, t_values_array_numba
def get_trades_based_amihud_lambda(log_ret: list,
dollar_volume: list) -> List[float]:
"""
Get Amihud lambda from trades data, p.288-289.
:param log_ret: (list) of log returns
:param dollar_volume: (list) of dollar volumes (price * size)
:return: (float) Amihud lambda for a bar
"""
X = np.array(dollar_volume).reshape(-1, 1)
y = np.abs(np.array(log_ret))
coef, std = get_betas(X, y)
t_value = coef[0] / std[0]
return [coef[0], t_value[0]]
def get_trades_based_hasbrouck_lambda(log_ret: list, dollar_volume: list,
aggressor_flags: list) -> List[float]:
"""
Get Hasbrouck lambda from trades data, p.289-290.
:param log_ret: (list) of log returns
:param dollar_volume: (list) of dollar volumes (price * size)
:param aggressor_flags: (list) of trade directions [-1, 1] (tick rule or aggressor side can be used to define)
:return: (list) Hasbrouck lambda for a bar and t value
"""
X = (np.sqrt(np.array(dollar_volume)) * np.array(aggressor_flags)).reshape(
-1, 1)
y = np.abs(np.array(log_ret))
coef, std = get_betas(X, y)
t_value = coef[0] / std[0]
return [coef[0], t_value[0]]
def get_trades_based_kyle_lambda(price_diff: list, volume: list,
aggressor_flags: list) -> List[float]:
"""
Get Kyle lambda from trades data, p.286-288.
:param price_diff: (list) of price diffs
:param volume: (list) of trades sizes
:param aggressor_flags: (list) of trade directions [-1, 1] (tick rule or aggressor side can be used to define)
:return: (list) Kyle lambda for a bar and t-value
"""
signed_volume = np.array(volume) * np.array(aggressor_flags)
X = np.array(signed_volume).reshape(-1, 1)
y = np.array(price_diff)
coef, std = get_betas(X, y)
t_value = coef[0] / std[0]
return [coef[0], t_value[0]]
def get_trades_based_amihud_lambda(log_ret: list,
dollar_volume: list) -> List[float]:
"""
Advances in Financial Machine Learning, p.288-289.
Get Amihud lambda from trades data
:param log_ret: (list) Log returns
:param dollar_volume: (list) Dollar volumes (price * size)
:return: (float) Amihud lambda for a bar
"""
X = np.array(dollar_volume).reshape(-1, 1)
y = np.abs(np.array(log_ret))
coef, std = get_betas(X, y)
t_value = coef[0] / std[0] if std[0] > 0 else np.array([0])
return [coef[0], t_value[0]]
def _get_dfc_for_t(series: pd.Series, molecule: list) -> pd.Series:
"""
Get Chow-Type Dickey-Fuller Test statistics for each index in molecule
:param series: (pd.Series) Series to test
:param molecule: (list) Dates to test
:return: (pd.Series) Statistics for each index from molecule
"""
dfc_series = pd.Series(index=molecule, dtype='float64')
for index in molecule:
series_diff = series.diff().dropna()
series_lag = series.shift(1).dropna()
series_lag[:index] = 0 # D_t* indicator: before t* D_t* = 0
y = series_diff.loc[series_lag.index].values
x = series_lag.values
coefs, coef_vars = get_betas(x.reshape(-1, 1), y)
b_estimate, b_var = coefs[0], coef_vars[0][0]
dfc_series[index] = b_estimate / (b_var**0.5)
return dfc_series
def test_sadf_test(self):
"""
Test get_sadf function
"""
log_prices = np.log(self.data.close)
lags_int = 5
lags_array = [1, 2, 5, 7]
min_length = 20
linear_sadf = get_sadf(log_prices,
model='linear',
add_const=True,
min_length=min_length,
lags=lags_int)
linear_sadf_no_const_lags_arr = get_sadf(log_prices,
model='linear',
add_const=False,
min_length=min_length,
lags=lags_array)
quadratic_sadf = get_sadf(log_prices,
model='quadratic',
add_const=True,
min_length=min_length,
lags=lags_int)
sm_poly_1_sadf = get_sadf(log_prices,
model='sm_poly_1',
add_const=True,
min_length=min_length,
lags=lags_int)
sm_poly_2_sadf = get_sadf(log_prices,
model='sm_poly_2',
add_const=True,
min_length=min_length,
lags=lags_int)
sm_power_sadf = get_sadf(log_prices,
model='sm_power',
add_const=True,
min_length=min_length,
lags=lags_int)
sm_exp_sadf = get_sadf(log_prices,
model='sm_exp',
add_const=True,
min_length=min_length,
lags=lags_int)
sm_power_sadf_phi = get_sadf(log_prices,
model='sm_power',
add_const=True,
min_length=min_length,
lags=lags_int,
phi=0.5)
sm_exp_sadf_phi = get_sadf(log_prices,
model='sm_exp',
add_const=True,
min_length=min_length,
lags=lags_int,
phi=0.5)
self.assertEqual(log_prices.shape[0] - min_length - lags_int - 1,
sm_power_sadf.shape[0]) # -1 for series_diff
self.assertEqual(log_prices.shape[0] - min_length - lags_int - 1,
linear_sadf.shape[0])
self.assertEqual(log_prices.shape[0] - min_length - lags_int - 1,
quadratic_sadf.shape[0])
self.assertEqual(log_prices.shape[0] - min_length - lags_int - 1,
sm_poly_1_sadf.shape[0])
self.assertEqual(log_prices.shape[0] - min_length - lags_int - 1,
sm_poly_2_sadf.shape[0])
self.assertEqual(log_prices.shape[0] - min_length - lags_int - 1,
sm_exp_sadf.shape[0])
self.assertEqual(log_prices.shape[0] - min_length - lags_int - 1,
sm_exp_sadf_phi.shape[0])
self.assertAlmostEqual(sm_power_sadf.mean(), 28.954, delta=1e-3)
self.assertAlmostEqual(sm_power_sadf.iloc[29], 17.369, delta=1e-3)
self.assertAlmostEqual(linear_sadf.mean(), -0.669, delta=1e-3)
self.assertAlmostEqual(linear_sadf[29], -0.717, delta=1e-3)
self.assertAlmostEqual(linear_sadf_no_const_lags_arr.mean(),
1.899,
delta=1e-3)
self.assertAlmostEqual(linear_sadf_no_const_lags_arr[29],
1.252,
delta=1e-3)
self.assertAlmostEqual(quadratic_sadf.mean(), -1.002, delta=1e-3)
self.assertAlmostEqual(quadratic_sadf[29], -1.460, delta=1e-3)
self.assertAlmostEqual(sm_poly_1_sadf.mean(), 26.033, delta=1e-3)
self.assertAlmostEqual(sm_poly_1_sadf[29], 8.350, delta=1e-3)
self.assertAlmostEqual(sm_poly_2_sadf.mean(), 26.031, delta=1e-3)
self.assertAlmostEqual(sm_poly_2_sadf[29], 8.353, delta=1e-3)
self.assertAlmostEqual(sm_exp_sadf.mean(), 28.916, delta=1e-3)
self.assertAlmostEqual(sm_exp_sadf[29], 17.100, delta=1e-3)
self.assertAlmostEqual(sm_power_sadf_phi.mean(), 1.4874, delta=1e-3)
self.assertAlmostEqual(sm_power_sadf_phi.iloc[29], 2.4564, delta=1e-3)
self.assertAlmostEqual(sm_exp_sadf_phi.mean(), 1.4787, delta=1e-3)
self.assertAlmostEqual(sm_exp_sadf_phi[29], 2.4183, delta=1e-3)
# Trivial series case.
ones_series = pd.Series(index=log_prices.index,
data=np.ones(shape=log_prices.shape[0]))
trivial_sadf = get_sadf(ones_series,
model='sm_power',
add_const=True,
min_length=min_length,
lags=lags_int,
phi=0.5)
self.assertTrue((trivial_sadf.unique() == [
-np.inf
]).all()) # All values should be -np.inf
# Test rubbish model argument.
self.assertRaises(ValueError,
get_sadf,
series=log_prices,
model='rubbish_string',
add_const=True,
min_length=min_length,
lags=lags_int)
# Assert that nans are parsed if singular matrix
singular_matrix = np.array([[1, 0, 0], [-1, 3, 3], [1, 2, 2]])
b_mean, b_var = get_betas(singular_matrix, singular_matrix)
self.assertTrue(b_mean, [np.nan])
self.assertTrue(b_var, [[np.nan, np.nan]])
def trend_scanning_labels(price_series: pd.Series,
t_events: list = None,
look_forward_window: int = 20,
min_sample_length: int = 5,
step: int = 1) -> pd.DataFrame:
"""
`Trend scanning <https://papers.ssrn.com/sol3/papers.cfm?abstract_id=3257419>`_ is both a classification and
regression labeling technique.
That can be used in the following ways:
1. Classification: By taking the sign of t-value for a given observation we can set {-1, 1} labels to define the
trends as either downward or upward.
2. Classification: By adding a minimum t-value threshold you can generate {-1, 0, 1} labels for downward, no-trend,
upward.
3. The t-values can be used as sample weights in classification problems.
4. Regression: The t-values can be used in a regression setting to determine the magnitude of the trend.
The output of this algorithm is a DataFrame with t1 (time stamp for the farthest observation), t-value, returns for
the trend, and bin.
:param price_series: (pd.Series) Close prices used to label the data set
:param t_events: (list) Filtered events, array of pd.Timestamps
:param look_forward_window: (int) Maximum look forward window used to get the trend value
:param min_sample_length: (int) Minimum sample length used to fit regression
:param step: (int) Optimal t-value index is searched every 'step' indices
:return: (pd.DataFrame) Consists of t1, t-value, ret, bin (label information). t1 - label endtime, tvalue,
ret - price change %, bin - label value based on price change sign
"""
# pylint: disable=invalid-name
if t_events is None:
t_events = price_series.index
t1_array = [] # Array of label end times
t_values_array = [] # Array of trend t-values
for index in t_events:
subset = price_series.loc[
index:].iloc[:look_forward_window] # Take t:t+L window
if subset.shape[0] >= look_forward_window:
# Loop over possible look-ahead windows to get the one which yields maximum t values for b_1 regression coef
max_abs_t_value = -np.inf # Maximum abs t-value of b_1 coefficient among l values
max_t_value_index = None # Index with maximum t-value
max_t_value = None # Maximum t-value signed
# Get optimal label end time value based on regression t-statistics
for forward_window in np.arange(min_sample_length, subset.shape[0],
step):
y_subset = subset.iloc[:forward_window].values.reshape(
-1, 1) # y{t}:y_{t+l}
# Array of [1, 0], [1, 1], [1, 2], ... [1, l] # b_0, b_1 coefficients
X_subset = np.ones((y_subset.shape[0], 2))
X_subset[:, 1] = np.arange(y_subset.shape[0])
# Get regression coefficients estimates
b_mean_, b_std_ = get_betas(X_subset, y_subset)
# Check if l gives the maximum t-value among all values {0...L}
t_beta_1 = (b_mean_[1] / np.sqrt(b_std_[1, 1]))[0]
if abs(t_beta_1) > max_abs_t_value:
max_abs_t_value = abs(t_beta_1)
max_t_value = t_beta_1
max_t_value_index = forward_window
# Store label information (t1, return)
label_endtime_index = subset.index[max_t_value_index - 1]
t1_array.append(label_endtime_index)
t_values_array.append(max_t_value)
else:
t1_array.append(None)
t_values_array.append(None)
labels = pd.DataFrame({
't1': t1_array,
't_value': t_values_array
},
index=t_events)
labels.loc[:, 'ret'] = price_series.loc[
labels.t1].values / price_series.loc[labels.index].values - 1
labels['bin'] = np.sign(labels.t_value)
return labels