def test_retrieving_intermittently_cached_intraday(tmpdir, provider):
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir,)
data = pd.DataFrame()
dates = generate_trading_days(
start_date=date.today() - timedelta(days=10),
end_date=date.today() - timedelta(days=1),
)
date_ranges = [
dates[1:2],
dates[-3:-2],
dates,
]
for date_range in date_ranges:
start_date = date_range[0]
end_date = date_range[-1]
data = retriever.retrieve_bar_data(
symbol="SPY",
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
validate_data_range(data=data, start_date=dates[0], end_date=dates[-1])
def test_retrieving_intermittently_cached_trades(tmpdir, provider):
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir,)
data = pd.DataFrame()
dates = generate_trading_days(
start_date=date(2020, 7, 21), end_date=date(2020, 7, 23),
)
date_ranges = [
dates[:1],
dates[-1:],
dates,
]
for date_range in date_ranges:
start_date = date_range[0]
end_date = date_range[-1]
contract = StockContract(symbol="SPY")
try:
data = retriever.retrieve_trades_data(
contract=contract, start_date=start_date, end_date=end_date,
)
except NotImplementedError:
return
validate_data_range(data=data, start_date=dates[0], end_date=dates[-1])
def test_retrieving_intermittently_cached_intraday(tmpdir, provider):
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir,)
data = pd.DataFrame()
dates = generate_trading_days(
start_date=date.today() - timedelta(days=10),
end_date=date.today() - timedelta(days=1),
)
date_ranges = [
dates[1:2],
dates[-3:-2],
dates,
]
for date_range in date_ranges:
start_date = date_range[0]
end_date = date_range[-1]
contract = StockContract(symbol="SPY")
try:
data = retriever.retrieve_bar_data(
contract=contract,
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
except NotImplementedError: # todo: fix
return
validate_data_range(data=data, start_date=dates[0], end_date=dates[-1])
def __init__(self, start_date: date, end_date: date, bar_size: timedelta):
hist_retriever = HistoricalRetriever(hist_data_dir=TEST_DATA_DIR)
self._sim_data = hist_retriever.retrieve_bar_data(
symbol="SPY",
bar_size=bar_size,
start_date=start_date,
end_date=end_date,
cache_only=True,
)
self._sim_idx = 0
def test_retrieve_non_cached_daily(tmpdir, provider):
start_date = date(2020, 4, 1)
end_date = date(2020, 4, 2)
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir,)
data = retriever.retrieve_bar_data(
symbol="SPY",
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
validate_data_range(data=data, start_date=start_date, end_date=end_date)
def test_retrieve_non_cached_intraday(tmpdir, provider):
start_date = date.today() - timedelta(days=7)
end_date = date.today() - timedelta(days=1)
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir,)
data = retriever.retrieve_bar_data(
symbol="SPY",
start_date=start_date,
end_date=end_date,
bar_size=timedelta(minutes=1),
)
validate_data_range(data=data, start_date=start_date, end_date=end_date)
assert np.isclose(len(data), 5 * 6.5 * 60, atol=7 * 60)
def test_retrieve_non_cached_trades_data(tmpdir, provider):
start_date = date(2020, 7, 22)
end_date = date(2020, 7, 23)
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir)
contract = StockContract(symbol="SPY")
try:
data = retriever.retrieve_trades_data(
contract=contract, start_date=start_date, end_date=end_date,
)
except NotImplementedError:
return
validate_data_range(data=data, start_date=start_date, end_date=end_date)
def __init__(
self,
starting_funds: float,
transaction_cost: float,
start_date: date,
end_date: date,
simulation_time_step: timedelta = timedelta(minutes=1),
real_time_per_tick: int = 0,
starting_positions: Optional[dict] = None,
hist_retriever: Optional[HistoricalRetriever] = None,
):
ABroker.__init__(self, simulation=True)
self._clock = SimulationClock(
start_date=start_date,
end_date=end_date,
simulation_time_step=simulation_time_step,
real_time_per_tick=real_time_per_tick,
)
if hist_retriever is None:
hist_retriever = HistoricalRetriever()
self._cash = starting_funds
self._positions = {}
if starting_positions is not None:
self._positions.update(**starting_positions)
self._hist_retriever = hist_retriever
self._abs_fee = transaction_cost
self._bars_callback_table = {}
self._local_cache = {}
self._hist_cache_only = None
def test_retriever_cached_daily():
start_date = date(2020, 4, 1)
end_date = date(2020, 4, 2)
retriever = HistoricalRetriever(hist_data_dir=TEST_DATA_DIR)
data = retriever.retrieve_bar_data(
symbol="SPY",
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
cache_only=True,
)
assert len(data) != 0
validate_data_range(data=data, start_date=start_date, end_date=end_date)
def test_retrieve_cached_trades():
start_date = date(2020, 6, 17)
end_date = date(2020, 6, 19)
retriever = HistoricalRetriever(hist_data_dir=TEST_DATA_DIR)
contract = StockContract(symbol="SPY")
data = retriever.retrieve_trades_data(
contract=contract,
start_date=start_date,
end_date=end_date,
cache_only=True,
)
assert len(data) != 0
validate_data_range(data=data, start_date=start_date, end_date=end_date)
def test_retriever_cached_tick():
start_date = date(2020, 6, 17)
end_date = date(2020, 6, 19)
retriever = HistoricalRetriever(hist_data_dir=TEST_DATA_DIR)
contract = StockContract(symbol="SCHW")
data = retriever.retrieve_bar_data(
contract=contract,
start_date=start_date,
end_date=end_date,
bar_size=timedelta(0),
cache_only=True,
)
assert len(data) != 0
validate_data_range(data=data, start_date=start_date, end_date=end_date)
def test_retrieving_intermittently_cached_daily(tmpdir, provider):
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir,)
start_date = date(2020, 3, 3)
end_date = date(2020, 3, 3)
retriever.retrieve_bar_data(
symbol="SPY",
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
start_date = date(2020, 3, 5)
end_date = date(2020, 3, 5)
retriever.retrieve_bar_data(
symbol="SPY",
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
start_date = date(2020, 3, 2)
end_date = date(2020, 3, 6)
data = retriever.retrieve_bar_data(
symbol="SPY",
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
validate_data_range(data=data, start_date=start_date, end_date=end_date)
def test_retrieve_non_cached_daily(tmpdir, provider):
start_date = date(2020, 4, 1)
end_date = date(2020, 4, 2)
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir,)
contract = StockContract(symbol="SPY")
try:
data = retriever.retrieve_bar_data(
contract=contract,
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
except NotImplementedError:
return
validate_data_range(data=data, start_date=start_date, end_date=end_date)
def sim_broker_15m():
broker = SimulationBroker(
starting_funds=1_000,
transaction_cost=1,
hist_retriever=HistoricalRetriever(hist_data_dir=TEST_DATA_DIR),
start_date=date(2020, 4, 6),
end_date=date(2020, 4, 7),
simulation_time_step=timedelta(minutes=15),
)
return broker
def test_retrieve_non_cached_trades_data_today_partial(tmpdir, provider):
end_date = date.today()
start_date = end_date - timedelta(days=1)
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir)
contract = StockContract(symbol="SPY")
try:
data = retriever.retrieve_trades_data(
contract=contract,
start_date=start_date,
end_date=end_date,
allow_partial=True,
rth=False,
)
except NotImplementedError:
return
validate_data_range(data=data, start_date=start_date, end_date=end_date)
assert data.iloc[-1].name.date() == date.today()
def test_simulation_broker_init():
broker = SimulationBroker(
starting_funds=1_000,
transaction_cost=1,
hist_retriever=HistoricalRetriever(hist_data_dir=TEST_DATA_DIR),
start_date=date(2020, 4, 6),
end_date=date(2020, 4, 6),
simulation_time_step=timedelta(minutes=15),
)
assert broker.acc_cash == 1_000
assert broker.get_position("SPY") == 0
assert broker.get_transaction_fee() == 1
def test_retrieve_non_cached_intraday(tmpdir, provider):
start_date = date.today() - timedelta(days=7)
end_date = date.today() - timedelta(days=1)
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir)
contract = StockContract(symbol="SPY")
try:
data = retriever.retrieve_bar_data(
contract=contract,
start_date=start_date,
end_date=end_date,
bar_size=timedelta(minutes=1),
)
except NotImplementedError:
return
validate_data_range(data=data, start_date=start_date, end_date=end_date)
assert (
np.isclose(len(data), 5 * 6.5 * 60, atol=7 * 60)
or len(data) == 4800 # for outside RTHs IB
)
def test_historical_bar_aggregator():
start_date = date(2020, 4, 6)
end_date = date(2020, 4, 7)
retriever = HistoricalRetriever(hist_data_dir=TEST_DATA_DIR)
contract = StockContract(symbol="SPY")
base_data = retriever.retrieve_bar_data(
contract=contract,
start_date=start_date,
end_date=end_date,
bar_size=timedelta(minutes=1),
cache_only=True,
)
aggregator = HistoricalAggregator(hist_data_dir=TEST_DATA_DIR)
agg_data = aggregator.aggregate_data(
contract=contract,
start_date=start_date,
end_date=end_date,
base_bar_size=timedelta(minutes=1),
target_bar_size=timedelta(minutes=5),
)
assert len(agg_data) == 156
assert agg_data.iloc[0]["open"] == base_data.iloc[0]["open"]
assert agg_data.iloc[0]["close"] == base_data.iloc[4]["close"]
assert agg_data.iloc[0]["high"] == base_data.iloc[:5]["high"].max()
assert agg_data.iloc[0]["low"] == base_data.iloc[:5]["low"].min()
assert agg_data.iloc[0]["volume"] == base_data.iloc[:5]["volume"].sum()
agg_data = aggregator.aggregate_data(
contract=contract,
start_date=start_date,
end_date=end_date,
base_bar_size=timedelta(minutes=1),
target_bar_size=timedelta(minutes=10),
)
assert len(agg_data) == 78
def __init__(
self,
historical_retriever: Optional[HistoricalRetriever] = None,
):
ADataStreamer.__init__(self)
ASimulationPiece.__init__(self)
if historical_retriever is None:
historical_retriever = HistoricalRetriever()
self._hist_retriever = historical_retriever
self._local_cache = {} # {contract: pd.DataFrame}
# {bar_size: {contract: {func: fn_kwargs}}}
self._bars_callback_table = {}
# {contract: {func: {"fn_kwargs": fn_kwargs, "price_type": price_type}}}
self._tick_callback_table = {}
# {contract: {func: fn_kwargs}}
self._greeks_callback_table = {}
self._hist_cache_only = None
def sim_broker_runner_and_streamer_1s():
sim_clock = SimulationClock(
start_date=date(2020, 6, 17),
end_date=date(2020, 6, 17),
simulation_time_step=timedelta(seconds=1),
)
streamer = SimulationDataStreamer(historical_retriever=HistoricalRetriever(
hist_data_dir=TEST_DATA_DIR), )
broker = SimulationBroker(
sim_streamer=streamer,
starting_funds={Currency.USD: 1_000},
transaction_cost=1,
)
sim_runner = SimulationRunner(
sim_clock=sim_clock,
data_providers=[streamer],
data_consumers=[broker],
)
return broker, sim_runner, streamer
def get_sme_sim_broker_daily(data_dir):
retriever = HistoricalRetriever(hist_data_dir=data_dir)
broker = SimulationBroker(
starting_funds=10_000,
transaction_cost=1,
hist_retriever=retriever,
start_date=date(2020, 2, 1),
end_date=date(2020, 3, 31),
simulation_time_step=timedelta(days=1),
)
trader = SMATrader(
broker=broker,
symbol="TEST",
bar_size=timedelta(days=1),
window=10,
sma_offset=0.1,
entry_n_shares=1,
)
trader.start()
return broker
def get_sme_sim_broker_intra_single_day(data_dir):
retriever = HistoricalRetriever(hist_data_dir=data_dir)
broker = SimulationBroker(
starting_funds=10_000,
transaction_cost=1,
hist_retriever=retriever,
start_date=date(2020, 4, 6),
end_date=date(2020, 4, 6),
simulation_time_step=timedelta(minutes=5),
)
trader = SMATrader(
broker=broker,
symbol="TEST",
bar_size=timedelta(minutes=5),
window=10,
sma_offset=0.1,
entry_n_shares=1,
exit_start=time(15, 50),
full_exit=time(15, 55),
)
trader.start()
return broker
def test_retrieving_intermittently_cached_daily(tmpdir, provider):
retriever = HistoricalRetriever(provider=provider, hist_data_dir=tmpdir)
start_date = date(2020, 3, 3)
end_date = date(2020, 3, 3)
contract = StockContract(symbol="SPY")
try:
retriever.retrieve_bar_data(
contract=contract,
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
except NotImplementedError: # todo: fix
return
start_date = date(2020, 3, 5)
end_date = date(2020, 3, 5)
retriever.retrieve_bar_data(
contract=contract,
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
start_date = date(2020, 3, 2)
end_date = date(2020, 3, 6)
data = retriever.retrieve_bar_data(
contract=contract,
start_date=start_date,
end_date=end_date,
bar_size=timedelta(days=1),
)
validate_data_range(data=data, start_date=start_date, end_date=end_date)
from datetime import timedelta, date
from algotradepy.historical.loaders import HistoricalRetriever
from algotradepy.historical.providers import YahooProvider
if __name__ == "__main__":
provider = YahooProvider()
hr = HistoricalRetriever(provider=provider)
end_date = date.today()
hr.retrieve_bar_data(
symbol="SPY",
bar_size=timedelta(days=1),
start_date=date(2000, 1, 1),
end_date=end_date,
)