async def save(
symbol: str,
new_qty: int,
last_op: str,
price: float,
indicators: Dict[Any, Any],
now: str,
) -> None:
db_trade = NewTrade(
algo_run_id=trading_data.open_order_strategy[symbol].algo_run.run_id,
symbol=symbol,
qty=new_qty,
operation=last_op,
price=price,
indicators=indicators,
)
await db_trade.save(
config.db_conn_pool,
str(now),
trading_data.stop_prices[symbol]
if symbol in trading_data.stop_prices
else 0.0,
trading_data.target_prices[symbol]
if symbol in trading_data.target_prices
else 0.0,
)
async def do_strategy_result(strategy: Strategy, symbol: str, now: datetime,
what: Dict) -> bool:
global portfolio_value
sign = (1 if
(what["side"] == "buy" and float(what["qty"]) > 0
or what["side"] == "sell" and float(what["qty"]) < 0) else -1)
try:
price = await calculate_execution_price(
symbol=symbol,
data_loader=strategy.data_loader,
what=what,
now=now)
if what["side"] == "buy":
await strategy.buy_callback(symbol, price, int(float(what["qty"])),
now)
elif what["side"] == "sell":
await strategy.sell_callback(symbol, price,
int(float(what["qty"])), now)
except Exception as e:
tlog(
f"do_strategy_result({symbol}, {what}, {now}) failed w/ {e}. operation not executed"
)
return False
try:
trading_data.positions[symbol] = trading_data.positions[
symbol] + sign * int(float(what["qty"]))
except KeyError:
trading_data.positions[symbol] = sign * int(float(what["qty"]))
trading_data.buy_time[symbol] = now.replace(second=0, microsecond=0)
trading_data.last_used_strategy[symbol] = strategy
db_trade = NewTrade(
algo_run_id=strategy.algo_run.run_id,
symbol=symbol,
qty=int(float(what["qty"])),
operation=what["side"],
price=price,
indicators={},
)
await db_trade.save(
config.db_conn_pool,
str(now),
trading_data.stop_prices[symbol]
if symbol in trading_data.stop_prices else None,
trading_data.target_prices[symbol]
if symbol in trading_data.target_prices else None,
)
return True
async def execute_portfolio(self, portfolio_id: str, df: df,
now: datetime) -> None:
tlog("Executing portfolio buys")
trader = trader_factory()
algo_run = await trader.create_session(self.name)
await DBPortfolio.associate_batch_id_to_profile(
portfolio_id, algo_run.batch_id)
orders = [
await trader.submit_order(
symbol=row.symbol,
qty=row.qty,
side="buy",
order_type="market",
time_in_force="day",
) for _, row in df.iterrows()
]
open_orders = []
while True:
for order in orders:
(
order_completed,
executed_price,
) = await trader.is_order_completed(order)
if order_completed:
db_trade = NewTrade(
algo_run_id=algo_run.run_id,
symbol=order.symbol,
qty=int(order.qty),
operation="buy",
price=executed_price,
indicators={},
)
await db_trade.save(
config.db_conn_pool,
str(now),
0.0,
0.0,
)
else:
open_orders.append(order)
if not len(open_orders):
break
await asyncio.sleep(5.0)
orders = open_orders
async def do_strategy_result(strategy: Strategy, symbol: str, now: datetime,
what: Dict) -> None:
global portfolio_value
if (what["side"] == "buy" and float(what["qty"]) > 0
or what["side"] == "sell" and float(what["qty"]) < 0):
try:
trading_data.positions[symbol] += int(float(what["qty"]))
except KeyError:
trading_data.positions[symbol] = int(float(what["qty"]))
trading_data.buy_time[symbol] = now.replace(second=0, microsecond=0)
else:
try:
trading_data.positions[symbol] -= int(float(what["qty"]))
except KeyError:
trading_data.positions[symbol] = -int(float(what["qty"]))
trading_data.last_used_strategy[symbol] = strategy
price = strategy.data_loader[symbol].close[now] # type: ignore
db_trade = NewTrade(
algo_run_id=strategy.algo_run.run_id,
symbol=symbol,
qty=int(float(what["qty"])),
operation=what["side"],
price=price,
indicators=trading_data.buy_indicators[symbol]
if what["side"] == "buy" else trading_data.sell_indicators[symbol],
)
await db_trade.save(
config.db_conn_pool,
str(now),
trading_data.stop_prices[symbol]
if symbol in trading_data.stop_prices else None,
trading_data.target_prices[symbol]
if symbol in trading_data.target_prices else None,
)
if what["side"] == "buy":
await strategy.buy_callback(symbol, price, int(float(what["qty"])))
elif what["side"] == "sell":
await strategy.sell_callback(symbol, price, int(float(what["qty"])))
async def liquidate(self):
for symbol in trading_data.positions:
if (trading_data.positions[symbol] != 0
and trading_data.last_used_strategy[symbol].type
== StrategyType.DAY_TRADE):
position = trading_data.positions[symbol]
minute_index = self.data_loader[symbol]["close"].index.get_loc(
self.now, method="nearest")
price = self.data_loader[symbol]["close"][minute_index]
tlog(f"[{self.end}]{symbol} liquidate {position} at {price}")
db_trade = NewTrade(
algo_run_id=trading_data.last_used_strategy[symbol].
algo_run.run_id, # type: ignore
symbol=symbol,
qty=int(position) if int(position) > 0 else -int(position),
operation="sell" if position > 0 else "buy",
price=price,
indicators={"liquidate": 1},
)
await db_trade.save(config.db_conn_pool,
str(self.now.to_pydatetime()))
async def next_minute(self) -> Tuple[bool, List[Optional[str]]]:
rc_msg: List[Optional[str]] = []
if self.now < self.end:
for i in range(0, len(self.scanners)):
if self.now == self.start or (
self.scanners[i].recurrence is not None
and self.scanners[i].recurrence.total_seconds() > 0 # type: ignore
and int((self.now - self.start).total_seconds() // 60) # type: ignore
% int(self.scanners[i].recurrence.total_seconds() // 60) # type: ignore
== 0
):
new_symbols = await self.scanners[i].run(self.now)
if new_symbols:
really_new = [
x for x in new_symbols if x not in self.symbols
]
if len(really_new) > 0:
print(
f"Loading data for {len(really_new)} symbols: {really_new}"
)
rc_msg.append(
f"Loaded data for {len(really_new)} symbols: {really_new}"
)
self.minute_history = {
**self.minute_history,
**(
market_data.get_historical_data_from_poylgon_for_symbols(
self.data_api,
really_new,
self.start - timedelta(days=7),
self.start + timedelta(days=1),
)
),
}
self.symbols += really_new
print(f"loaded data for {len(really_new)} stocks")
for symbol in self.symbols:
try:
for strategy in trading_data.strategies:
try:
minute_index = self.minute_history[symbol][
"close"
].index.get_loc(self.now, method="nearest")
except Exception as e:
print(f"[Exception] {self.now} {symbol} {e}")
print(self.minute_history[symbol]["close"][-100:])
continue
price = self.minute_history[symbol]["close"][
minute_index
]
if symbol not in trading_data.positions:
trading_data.positions[symbol] = 0
do, what = await strategy.run(
symbol,
True,
int(trading_data.positions[symbol]),
self.minute_history[symbol][: minute_index + 1],
self.now,
self.portfolio_value,
debug=False, # type: ignore
backtesting=True,
)
if do:
if (
what["side"] == "buy"
and float(what["qty"]) > 0
or what["side"] == "sell"
and float(what["qty"]) < 0
):
trading_data.positions[symbol] += int(
float(what["qty"])
)
trading_data.buy_time[
symbol
] = self.now.replace(second=0, microsecond=0)
else:
trading_data.positions[symbol] -= int(
float(what["qty"])
)
trading_data.last_used_strategy[symbol] = strategy
rc_msg.append(
f"[{self.now}][{strategy.name}] {what['side']} {what['qty']} of {symbol} @ {price}"
)
db_trade = NewTrade(
algo_run_id=strategy.algo_run.run_id,
symbol=symbol,
qty=int(float(what["qty"])),
operation=what["side"],
price=price,
indicators=trading_data.buy_indicators[symbol]
if what["side"] == "buy"
else trading_data.sell_indicators[symbol],
)
await db_trade.save(
config.db_conn_pool,
str(self.now.to_pydatetime()),
trading_data.stop_prices[symbol],
trading_data.target_prices[symbol],
)
if what["side"] == "buy":
await strategy.buy_callback(
symbol, price, int(float(what["qty"]))
)
break
elif what["side"] == "sell":
await strategy.sell_callback(
symbol, price, int(float(what["qty"]))
)
break
except Exception as e:
print(f"[Exception] {self.now} {symbol} {e}")
traceback.print_exc()
self.now += timedelta(minutes=1)
return True, rc_msg
else:
return False, []
async def backtest_symbol(
symbol: str, scanner_start_time: datetime
) -> None:
est = pytz.timezone("America/New_York")
scanner_start_time = (
pytz.utc.localize(scanner_start_time).astimezone(est)
if scanner_start_time.tzinfo is None
else scanner_start_time
)
start_time = pytz.utc.localize(start).astimezone(est)
if scanner_start_time > start_time + duration:
print(
f"{symbol} picked too late at {scanner_start_time} ({start_time}, {duration})"
)
return
start_time = scanner_start_time
if start_time.second > 0:
start_time = start_time.replace(second=0, microsecond=0)
print(
f"--> back-testing {symbol} from {str(start_time)} duration {duration}"
)
if debug_symbols and symbol in debug_symbols:
print("--> using DEBUG mode")
re_try = 3
while re_try > 0:
# load historical data
try:
symbol_data = data_api.polygon.historic_agg_v2(
symbol,
1,
"minute",
_from=str(start_time - timedelta(days=8)),
to=str(start_time + timedelta(days=1)),
limit=10000,
).df
except HTTPError as e:
tlog(f"Received HTTP error {e} for {symbol}")
return
if len(symbol_data) < 100:
tlog(f"not enough data-points for {symbol}")
return
add_daily_vwap(
symbol_data,
debug=debug_symbols and symbol in debug_symbols,
)
market_data.minute_history[symbol] = symbol_data
print(
f"loaded {len(market_data.minute_history[symbol].index)} agg data points"
)
position: int = 0
try:
minute_index = symbol_data["close"].index.get_loc(
start_time, method="nearest"
)
break
except (Exception, ValueError) as e:
print(f"[EXCEPTION] {e} - trying to reload-data. ")
re_try -= 1
new_now = symbol_data.index[minute_index]
print(f"start time with data {new_now}")
price = 0.0
last_run_id = None
# start_time + duration
while (
new_now < config.market_close
and minute_index < symbol_data.index.size - 1
):
if symbol_data.index[minute_index] != new_now:
print(
"mismatch!", symbol_data.index[minute_index], new_now
)
print(
symbol_data["close"][
minute_index - 10 : minute_index + 1
]
)
raise Exception()
price = symbol_data["close"][minute_index]
for strategy in trading_data.strategies:
if debug_symbols and symbol in debug_symbols:
print(
f"Execute strategy {strategy.name} on {symbol} at {new_now}"
)
do, what = await strategy.run(
symbol,
True,
position,
symbol_data[: minute_index + 1],
new_now,
portfolio_value,
debug=debug_symbols and symbol in debug_symbols, # type: ignore
backtesting=True,
)
if do:
if (
what["side"] == "buy"
and float(what["qty"]) > 0
or what["side"] == "sell"
and float(what["qty"]) < 0
):
position += int(float(what["qty"]))
trading_data.buy_time[symbol] = new_now.replace(
second=0, microsecond=0
)
else:
position -= int(float(what["qty"]))
trading_data.last_used_strategy[symbol] = strategy
db_trade = NewTrade(
algo_run_id=strategy.algo_run.run_id,
symbol=symbol,
qty=int(float(what["qty"])),
operation=what["side"],
price=price,
indicators=trading_data.buy_indicators[symbol]
if what["side"] == "buy"
else trading_data.sell_indicators[symbol],
)
await db_trade.save(
config.db_conn_pool,
str(new_now),
trading_data.stop_prices[symbol],
trading_data.target_prices[symbol],
)
if what["side"] == "buy":
await strategy.buy_callback(
symbol, price, int(float(what["qty"]))
)
break
elif what["side"] == "sell":
await strategy.sell_callback(
symbol, price, int(float(what["qty"]))
)
break
last_run_id = strategy.algo_run.run_id
minute_index += 1
new_now = symbol_data.index[minute_index]
if position:
if (
trading_data.last_used_strategy[symbol].type
== StrategyType.DAY_TRADE
):
tlog(
f"[{new_now}]{symbol} liquidate {position} at {price}"
)
db_trade = NewTrade(
algo_run_id=last_run_id, # type: ignore
symbol=symbol,
qty=int(position)
if int(position) > 0
else -int(position),
operation="sell" if position > 0 else "buy",
price=price,
indicators={"liquidate": 1},
)
await db_trade.save(
config.db_conn_pool,
str(symbol_data.index[minute_index - 1]),
)
async def backtest_symbol(
data_loader: DataLoader,
portfolio_value: float,
symbol: str,
start: datetime,
duration: timedelta,
scanner_start_time: datetime,
debug_symbol: bool = False,
) -> None:
est = pytz.timezone("America/New_York")
scanner_start_time = (pytz.utc.localize(scanner_start_time).astimezone(est)
if scanner_start_time.tzinfo is None else
scanner_start_time)
start_time = pytz.utc.localize(start).astimezone(est)
if scanner_start_time > start_time + duration:
print(
f"{symbol} picked too late at {scanner_start_time} ({start_time}, {duration})"
)
return
start_time = scanner_start_time
if start_time.second > 0:
start_time = start_time.replace(second=0, microsecond=0)
print(
f"--> back-testing {symbol} from {str(start_time)} duration {duration}"
)
if debug_symbol:
print("--> using DEBUG mode")
symbol_data = pd.DataFrame(data_loader[symbol][start_time:start_time +
duration] # type: ignore
)
add_daily_vwap(
symbol_data,
debug=debug_symbol,
)
print(
f"loaded {len(symbol_data)} agg data points({start_time}-{start_time + duration})"
)
minute_index = symbol_data["close"].index.get_loc(start_time,
method="nearest")
position: int = 0
new_now = symbol_data.index[minute_index]
print(f"start time with data {new_now}")
price = 0.0
last_run_id = None
# start_time + duration
rejected: Dict[str, List] = {}
while (new_now < config.market_close
and minute_index < symbol_data.index.size - 1):
if symbol_data.index[minute_index] != new_now:
print("mismatch!", symbol_data.index[minute_index], new_now)
print(symbol_data["close"][minute_index - 10:minute_index + 1])
raise Exception()
price = symbol_data["close"][minute_index]
for strategy in trading_data.strategies:
if debug_symbol:
print(
f"Execute strategy {strategy.name} on {symbol} at {new_now}"
)
if symbol in rejected.get(strategy.name, []):
continue
try:
do, what = await strategy.run(
symbol,
True,
position,
symbol_data[:minute_index + 1],
new_now,
portfolio_value,
debug=debug_symbol, # type: ignore
backtesting=True,
)
except Exception as e:
traceback.print_exc()
tlog(
f"[ERROR] exception {e} on symbol {symbol} @ {strategy.name}"
)
continue
if do:
if (what["side"] == "buy" and float(what["qty"]) > 0
or what["side"] == "sell" and float(what["qty"]) < 0):
position += int(float(what["qty"]))
trading_data.buy_time[symbol] = new_now.replace(
second=0, microsecond=0)
else:
position -= int(float(what["qty"]))
trading_data.last_used_strategy[symbol] = strategy
db_trade = NewTrade(
algo_run_id=strategy.algo_run.run_id,
symbol=symbol,
qty=int(float(what["qty"])),
operation=what["side"],
price=price,
indicators=trading_data.buy_indicators[symbol]
if what["side"] == "buy" else
trading_data.sell_indicators[symbol],
)
await db_trade.save(
config.db_conn_pool,
str(new_now),
trading_data.stop_prices[symbol],
trading_data.target_prices[symbol],
)
if what["side"] == "buy":
await strategy.buy_callback(symbol, price,
int(float(what["qty"])))
break
elif what["side"] == "sell":
await strategy.sell_callback(symbol, price,
int(float(what["qty"])))
break
elif what.get("reject", False):
if strategy.name in rejected:
rejected[strategy.name].append(symbol)
else:
rejected[strategy.name] = [symbol]
last_run_id = strategy.algo_run.run_id
minute_index += 1
new_now = symbol_data.index[minute_index]
if position:
if (trading_data.last_used_strategy[symbol].type ==
StrategyType.DAY_TRADE):
tlog(f"[{new_now}]{symbol} liquidate {position} at {price}")
db_trade = NewTrade(
algo_run_id=last_run_id, # type: ignore
symbol=symbol,
qty=int(position) if int(position) > 0 else -int(position),
operation="sell" if position > 0 else "buy",
price=price,
indicators={"liquidate": 1},
)
await db_trade.save(
config.db_conn_pool,
str(symbol_data.index[minute_index - 1]),
)