def __init__(
self,
symbols: List[str],
portfolio_size: float,
rank_days: int,
stock_count: int,
volatility_threshold: float,
data_loader: DataLoader,
trader: Trader,
debug=False,
):
try:
self.rank_days = rank_days
self.debug = debug
self.portfolio_size = portfolio_size
tlog(f"data_loader:{data_loader}")
self.data_loader = DataLoader(TimeScale.day)
self.symbols = symbols
self.stock_count = stock_count
self.volatility_threshold = volatility_threshold
self.trader: Trader = trader
except Exception:
raise ValueError(
"[ERROR] Miner must receive all valid parameter(s)"
)
self.portfolio: df = df(columns=["symbol", "slope", "r", "score"])
async def backtest_main(
uid: str,
from_date: date,
to_date: date,
scale: TimeScale,
tradeplan: Dict,
scanners: Optional[List],
strategies: Optional[List],
) -> None:
tlog(
f"Starting back-test from {from_date} to {to_date} with time scale {scale}"
)
global portfolio_value
if "portfolio_value" in tradeplan:
portfolio_value = tradeplan["portfolio_value"]
else:
portfolio_value = 100000
await create_db_connection()
data_loader = DataLoader(scale)
trade_api = tradeapi.REST(key_id=config.alpaca_api_key,
secret_key=config.alpaca_api_secret)
scanners = await create_scanners(data_loader, tradeplan["scanners"],
scanners)
strategies = await create_strategies(uid, tradeplan["strategies"],
data_loader, strategies)
calendars = trade_api.get_calendar(str(from_date), str(to_date))
symbols: Dict = {}
for day in calendars:
day_start = day.date.replace(
hour=day.open.hour,
minute=day.open.minute,
tzinfo=timezone("America/New_York"),
)
day_end = day.date.replace(
hour=day.close.hour,
minute=day.close.minute,
tzinfo=timezone("America/New_York"),
)
config.market_open = day_start
config.market_close = day_end
current_time = day_start
data_loader = DataLoader()
while current_time < day_end:
symbols = await do_scanners(current_time, scanners, symbols)
for strategy in strategies:
strategy_symbols = list(
set(symbols.get("_all", [])).union(
set(symbols.get(strategy.name, []))))
await do_strategy(data_loader, current_time, strategy,
strategy_symbols)
current_time += timedelta(seconds=scale.value)
def test_apple_stock_price_open_range_date_int_min_open() -> bool:
print("test_apple_stock_price_close_range_date_int_min_open")
config.data_connector = DataConnectorType.finnhub
dl = DataLoader(TimeScale.minute, DataConnectorType.finnhub)
last_price_range = dl["AAPL"].open["2020-10-05":] # type:ignore
print(last_price_range)
return True
def test_get_symbols_alpaca() -> bool:
print("test_get_symbols_alpaca")
dl = DataLoader(TimeScale.minute, connector=DataConnectorType.alpaca)
tickers = dl.data_api.get_symbols()
print(len(tickers))
return True
def test_get_symbols_finnhub() -> bool:
print("test_get_symbols_finnhub")
dl = DataLoader(TimeScale.minute, DataConnectorType.finnhub)
tickers = dl.data_api.get_symbols()
print(len(tickers))
return True
def __init__(self, conf_dict: Dict):
self.uid = str(uuid.uuid4())
self.conf_dict = conf_dict
config.portfolio_value = self.conf_dict.get("portfolio_value", None)
self.data_loader = DataLoader()
self.scanners: List[Scanner] = []
async def run(self) -> bool:
trader = trader_factory()
self.trend_logic = TrendLogic(
symbols=await sp500_historical_constituents(datetime.today()),
portfolio_size=self.portfolio_size,
rank_days=self.rank_days,
debug=self.debug,
stock_count=self.stock_count,
volatility_threshold=self.volatility_threshold,
data_loader=DataLoader(),
trader=trader,
)
if self.debug:
tlog(f"symbols: {self.trend_logic.symbols}")
df = (await self.trend_logic.run_short(
nyc.localize(datetime.utcnow()) +
timedelta(days=1)) if self.short else await self.trend_logic.run(
nyc.localize(datetime.utcnow()) + timedelta(days=1)))
portfolio_id = await self.save_portfolio()
await self.display_portfolio(df)
# await self.execute_portfolio(
# portfolio_id, df, nyc.localize(datetime.utcnow())
# )
print(
"-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
)
tlog(f"PORTFOLIO_ID:{portfolio_id}")
print(
"-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
)
return True
def test_apple_stock_price_range_date_day() -> bool:
print("test_apple_stock_price_range_date_day")
dl = DataLoader(TimeScale.day, connector=DataConnectorType.polygon)
last_price_range = dl["AAPL"]["2020-10-05":"2020-10-08"] # type:ignore
print(last_price_range)
return True
def test_apple_stock_current_price_range_int_day() -> bool:
print("test_apple_stock_current_price_range_int_day")
dl = DataLoader(TimeScale.day, connector=DataConnectorType.polygon)
last_price_range = dl["AAPL"].close[-6:-1] # type:ignore
print(last_price_range)
return True
def calc_batch_returns(batch_id: str) -> pd.DataFrame:
loop = asyncio.get_event_loop()
portfolio = loop.run_until_complete(Portfolio.load_by_batch_id(batch_id))
data_loader = DataLoader()
trades = load_trades_by_batch_id(batch_id)
start_date = trades.client_time.min().date()
end_date = trades.client_time.max().date()
trader = trader_factory()()
td = trader.get_trading_days(start_date=start_date, end_date=end_date)
td["equity"] = 0.0
td["cash"] = portfolio.portfolio_size
num_symbols = len(trades.symbol.unique().tolist())
for c, symbol in enumerate(trades.symbol.unique().tolist(), start=1):
print(f"{symbol} ({c}/{num_symbols})")
symbol_trades = trades[trades.symbol == symbol].sort_values(
by="client_time"
)
calc_symbol_trades_returns(symbol, symbol_trades, td, data_loader)
td["totals"] = td["equity"] + td["cash"]
# td["date"] = pd.to_datetime(td.index)
# td = td.set_index("date")
return pd.DataFrame(td, columns=["equity", "cash", "totals"])
def test_apple_stock_price_range_int_minute() -> bool:
print("test_apple_stock_price_range_int_minute")
dl = DataLoader(TimeScale.minute, connector=DataConnectorType.polygon)
last_price_range = dl["AAPL"][-5:-1] # type:ignore
print(last_price_range)
return True
def test_apple_stock_price_range_date_int_day() -> bool:
print("test_apple_stock_price_range_date_int_day")
dl = DataLoader(TimeScale.day)
last_price_range = dl["AAPL"]["2020-10-05":-1] # type:ignore
print(last_price_range)
return True
def calc_portfolio_returns(portfolio_id: str) -> pd.DataFrame:
loop = asyncio.get_event_loop()
_ = loop.run_until_complete(Portfolio.load_by_portfolio_id(portfolio_id))
account_id, initial_account_size = loop.run_until_complete(
Portfolio.load_details(portfolio_id)
)
data_loader = DataLoader()
trades = load_trades_by_portfolio(portfolio_id)
start_date = trades.client_time.min().date()
end_date = trades.client_time.max().date()
trader = trader_factory()()
td = trader.get_trading_days(start_date=start_date, end_date=end_date)
td["equity"] = 0.0
cash_df = get_cash(account_id, initial_account_size)
symbols = trades.symbol.unique().tolist()
for i in tqdm(range(len(symbols))):
symbol = symbols[i]
symbol_trades = trades[trades.symbol == symbol].sort_values(
by="client_time"
)
calc_symbol_trades_returns(symbol, symbol_trades, td, data_loader)
td = td.join(cash_df)
td = td.fillna(method="ffill")
td["totals"] = td["equity"] + td["cash"]
return pd.DataFrame(td, columns=["equity", "cash", "totals"])
def test_apple_stock_price_range_int_minute() -> bool:
print("test_apple_stock_close_price_range_str_minute")
dl = DataLoader(TimeScale.minute)
last_price_range = dl["AAPL"][-5:-1] # type:ignore
print(last_price_range)
return True
async def scanners_runner(scanners_conf: Dict, queue: mp.Queue,
trader: Trader) -> None:
print("** scanners_runner() task starting **")
scanners: List[Scanner] = await create_scanners(trader, DataLoader(),
scanners_conf)
scanner_tasks = [
asyncio.create_task(scanner_runner(scanner, queue))
for scanner in scanners
]
try:
await asyncio.gather(
*scanner_tasks,
return_exceptions=True,
)
except asyncio.CancelledError:
tlog(
"scanners_runner.scanners_runner() cancelled, closing scanner tasks"
)
for task in scanner_tasks:
tlog(
f"scanners_runner.scanners_runner() requesting task {task.get_name()} to cancel"
)
task.cancel()
try:
await task
except asyncio.CancelledError:
tlog(
"scanners_runner.scanners_runner() task is cancelled now")
finally:
queue.close()
tlog("scanners_runner.scanners_runner() done.")
def test_apple_stock_price_range_int_day() -> bool:
print("test_apple_stock_price_range_int_day")
dl = DataLoader(TimeScale.day, connector=DataConnectorType.alpaca)
last_price_range = dl["AAPL"][-5:-1] # type:ignore
print(last_price_range)
return True
async def consumer_async_main(
queue: Queue,
symbols: List[str],
unique_id: str,
strategies_conf: Dict,
):
await create_db_connection(str(config.dsn))
data_loader = DataLoader()
if symbols:
try:
trending_db = TrendingTickers(unique_id)
await trending_db.save(symbols)
except Exception as e:
tlog(
f"Exception in consumer_async_main() while storing symbols to DB:{type(e).__name__} with args {e.args}"
)
exc_info = sys.exc_info()
lines = traceback.format_exception(*exc_info)
for line in lines:
tlog(f"error: {line}")
traceback.print_exception(*exc_info)
del exc_info
trader = trader_factory()()
config.market_open, config.market_close = trader.get_market_schedule()
tlog(
f"market open:{config.market_open} market close:{config.market_close}")
loaded = await create_strategies(
batch_id=unique_id,
symbols=symbols,
trader=trader,
data_loader=data_loader,
strategies_conf=strategies_conf,
)
if symbols and loaded != len(symbols):
tlog(
f"[ERROR] Consumer process loaded only {loaded} out of {len(symbols)} open positions. HINT: make sure that your tradeplan.toml file includes all strategies from previous trading session."
)
queue_consumer_task = asyncio.create_task(
queue_consumer(queue, data_loader, trader))
liquidate_task = asyncio.create_task(liquidator(trader))
periodic_runner_task = asyncio.create_task(
periodic_runner(data_loader, trader))
tear_down = asyncio.create_task(
teardown_task(trader, [queue_consumer_task, periodic_runner_task]))
await asyncio.gather(
tear_down,
liquidate_task,
queue_consumer_task,
periodic_runner_task,
return_exceptions=True,
)
tlog("consumer_async_main() completed")
def test_apple_stock_close_price_range_int_str_minute() -> bool:
print("test_apple_stock_close_price_range_str_minute")
dl = DataLoader(TimeScale.minute, connector=DataConnectorType.polygon)
last_price_range = dl["AAPL"].close[-5:str(datetime.now()) # type:ignore
]
print(last_price_range)
return True
def test_apple_stock_close_price_range_str_minute_int() -> bool:
print("test_apple_stock_close_price_range_str_minute")
dl = DataLoader(TimeScale.minute, connector=DataConnectorType.polygon)
last_price_range = dl["AAPL"].close["2021-01-05 09:45:00":-1 # type:ignore
] # type:ignore
print(last_price_range)
return True
def test_get_symbols_polygon() -> bool:
print("test_get_symbols_polygon")
config.data_connector = DataConnectorType.polygon
dl = DataLoader(TimeScale.minute, connector=DataConnectorType.polygon)
tickers = dl.data_api.get_symbols()
print(len(tickers))
return True
def test_apple_stock_price_range_int_minute() -> bool:
print("test_apple_stock_close_price_range_str_minute")
config.data_connector = DataConnectorType.finnhub
dl = DataLoader(TimeScale.minute, DataConnectorType.finnhub)
last_price_range = dl["AAPL"][-5:-1] # type:ignore
print(last_price_range)
return True
def test_apple_stock_price_range_date_day_mixed() -> bool:
print("test_apple_stock_price_range_date_day_mixed")
dl = DataLoader(TimeScale.day, connector=DataConnectorType.polygon)
d1 = date(year=2021, month=2, day=1)
last_price_range = dl["AAPL"][d1:"2021-02-02"] # type:ignore
print(last_price_range)
return True
def test_apple_stock_price_open_date() -> bool:
print("test_apple_stock_price_open_date")
dl = DataLoader(TimeScale.minute, connector=DataConnectorType.polygon)
d1 = date(year=2021, month=2, day=1)
last_price_range = dl["AAPL"].open[d1]
print(last_price_range)
return True
def test_apple_stock_price_open_str() -> bool:
print("test_apple_stock_price_open_str")
dl = DataLoader(TimeScale.minute)
d1 = date(year=2021, month=2, day=1)
last_price_range = dl["AAPL"].open["2021-02-02 09:45:00"]
print(last_price_range)
return True
def test_apple_stock_price_range_date_int_day() -> bool:
print("test_apple_stock_price_range_date_int_day")
config.data_connector = DataConnectorType.finnhub
dl = DataLoader(TimeScale.day, DataConnectorType.finnhub)
last_price_range = dl["AAPL"]["2020-10-05":-1] # type:ignore
print(last_price_range)
return True
def test_negative_current_price() -> bool:
dl = DataLoader(TimeScale.minute, connector=DataConnectorType.polygon)
try:
dl["DFGDFGDFG"].close[-1]
except ValueError:
return True
return False
def test_apple_stock_price_range_date_min_mixed() -> bool:
print("test_apple_stock_price_range_date_min_mixed")
dl = DataLoader(TimeScale.minute, connector=DataConnectorType.polygon)
d1 = date(year=2021, month=2, day=1)
last_price_range = dl["AAPL"][d1:"2021-02-02"].between_time( # type:ignore
"9:30", "16:00") # type:ignore
print(last_price_range)
return True
def test_create_data_loader_types() -> bool:
for data_connector in DataConnectorType:
config.data_connector = data_connector
for scale in TimeScale:
if not DataLoader(scale=scale):
return False
config.data_connector = DataConnectorType.alpaca
return True
def test_apple_stock_price_range_date_min_open() -> bool:
print("test_apple_stock_price_range_date_min_open")
dl = DataLoader(TimeScale.minute)
try:
last_price_range = dl["AAPL"][:] # type:ignore
print(last_price_range)
except ValueError:
return True
return True
def test_apple_stock_price_open_range_date_min_open() -> bool:
print("test_apple_stock_price_open_range_date_min_open")
dl = DataLoader(TimeScale.minute, connector=DataConnectorType.polygon)
try:
last_price_range = dl["AAPL"].open[:] # type:ignore
print(last_price_range)
except ValueError:
return True
return True
