Example #1
0
async def save(
    symbol: str,
    new_qty: int,
    last_op: str,
    price: float,
    indicators: Dict[Any, Any],
    now: str,
) -> None:
    db_trade = NewTrade(
        algo_run_id=trading_data.open_order_strategy[symbol].algo_run.run_id,
        symbol=symbol,
        qty=new_qty,
        operation=last_op,
        price=price,
        indicators=indicators,
    )

    await db_trade.save(
        config.db_conn_pool,
        str(now),
        trading_data.stop_prices[symbol]
        if symbol in trading_data.stop_prices
        else 0.0,
        trading_data.target_prices[symbol]
        if symbol in trading_data.target_prices
        else 0.0,
    )
Example #2
0
async def do_strategy_result(strategy: Strategy, symbol: str, now: datetime,
                             what: Dict) -> bool:
    global portfolio_value

    sign = (1 if
            (what["side"] == "buy" and float(what["qty"]) > 0
             or what["side"] == "sell" and float(what["qty"]) < 0) else -1)

    try:
        price = await calculate_execution_price(
            symbol=symbol,
            data_loader=strategy.data_loader,
            what=what,
            now=now)

        if what["side"] == "buy":
            await strategy.buy_callback(symbol, price, int(float(what["qty"])),
                                        now)
        elif what["side"] == "sell":
            await strategy.sell_callback(symbol, price,
                                         int(float(what["qty"])), now)
    except Exception as e:
        tlog(
            f"do_strategy_result({symbol}, {what}, {now}) failed w/ {e}. operation not executed"
        )
        return False

    try:
        trading_data.positions[symbol] = trading_data.positions[
            symbol] + sign * int(float(what["qty"]))
    except KeyError:
        trading_data.positions[symbol] = sign * int(float(what["qty"]))
    trading_data.buy_time[symbol] = now.replace(second=0, microsecond=0)
    trading_data.last_used_strategy[symbol] = strategy

    db_trade = NewTrade(
        algo_run_id=strategy.algo_run.run_id,
        symbol=symbol,
        qty=int(float(what["qty"])),
        operation=what["side"],
        price=price,
        indicators={},
    )

    await db_trade.save(
        config.db_conn_pool,
        str(now),
        trading_data.stop_prices[symbol]
        if symbol in trading_data.stop_prices else None,
        trading_data.target_prices[symbol]
        if symbol in trading_data.target_prices else None,
    )

    return True
Example #3
0
    async def execute_portfolio(self, portfolio_id: str, df: df,
                                now: datetime) -> None:
        tlog("Executing portfolio buys")
        trader = trader_factory()

        algo_run = await trader.create_session(self.name)
        await DBPortfolio.associate_batch_id_to_profile(
            portfolio_id, algo_run.batch_id)

        orders = [
            await trader.submit_order(
                symbol=row.symbol,
                qty=row.qty,
                side="buy",
                order_type="market",
                time_in_force="day",
            ) for _, row in df.iterrows()
        ]

        open_orders = []
        while True:
            for order in orders:
                (
                    order_completed,
                    executed_price,
                ) = await trader.is_order_completed(order)
                if order_completed:
                    db_trade = NewTrade(
                        algo_run_id=algo_run.run_id,
                        symbol=order.symbol,
                        qty=int(order.qty),
                        operation="buy",
                        price=executed_price,
                        indicators={},
                    )
                    await db_trade.save(
                        config.db_conn_pool,
                        str(now),
                        0.0,
                        0.0,
                    )
                else:
                    open_orders.append(order)
            if not len(open_orders):
                break
            await asyncio.sleep(5.0)
            orders = open_orders
async def do_strategy_result(strategy: Strategy, symbol: str, now: datetime,
                             what: Dict) -> None:
    global portfolio_value
    if (what["side"] == "buy" and float(what["qty"]) > 0
            or what["side"] == "sell" and float(what["qty"]) < 0):
        try:
            trading_data.positions[symbol] += int(float(what["qty"]))
        except KeyError:
            trading_data.positions[symbol] = int(float(what["qty"]))
        trading_data.buy_time[symbol] = now.replace(second=0, microsecond=0)
    else:
        try:
            trading_data.positions[symbol] -= int(float(what["qty"]))
        except KeyError:
            trading_data.positions[symbol] = -int(float(what["qty"]))

    trading_data.last_used_strategy[symbol] = strategy

    price = strategy.data_loader[symbol].close[now]  # type: ignore
    db_trade = NewTrade(
        algo_run_id=strategy.algo_run.run_id,
        symbol=symbol,
        qty=int(float(what["qty"])),
        operation=what["side"],
        price=price,
        indicators=trading_data.buy_indicators[symbol]
        if what["side"] == "buy" else trading_data.sell_indicators[symbol],
    )

    await db_trade.save(
        config.db_conn_pool,
        str(now),
        trading_data.stop_prices[symbol]
        if symbol in trading_data.stop_prices else None,
        trading_data.target_prices[symbol]
        if symbol in trading_data.target_prices else None,
    )

    if what["side"] == "buy":
        await strategy.buy_callback(symbol, price, int(float(what["qty"])))
    elif what["side"] == "sell":
        await strategy.sell_callback(symbol, price, int(float(what["qty"])))
 async def liquidate(self):
     for symbol in trading_data.positions:
         if (trading_data.positions[symbol] != 0
                 and trading_data.last_used_strategy[symbol].type
                 == StrategyType.DAY_TRADE):
             position = trading_data.positions[symbol]
             minute_index = self.data_loader[symbol]["close"].index.get_loc(
                 self.now, method="nearest")
             price = self.data_loader[symbol]["close"][minute_index]
             tlog(f"[{self.end}]{symbol} liquidate {position} at {price}")
             db_trade = NewTrade(
                 algo_run_id=trading_data.last_used_strategy[symbol].
                 algo_run.run_id,  # type: ignore
                 symbol=symbol,
                 qty=int(position) if int(position) > 0 else -int(position),
                 operation="sell" if position > 0 else "buy",
                 price=price,
                 indicators={"liquidate": 1},
             )
             await db_trade.save(config.db_conn_pool,
                                 str(self.now.to_pydatetime()))
Example #6
0
    async def next_minute(self) -> Tuple[bool, List[Optional[str]]]:
        rc_msg: List[Optional[str]] = []
        if self.now < self.end:
            for i in range(0, len(self.scanners)):
                if self.now == self.start or (
                    self.scanners[i].recurrence is not None
                    and self.scanners[i].recurrence.total_seconds() > 0  # type: ignore
                    and int((self.now - self.start).total_seconds() // 60)  # type: ignore
                    % int(self.scanners[i].recurrence.total_seconds() // 60)  # type: ignore
                    == 0
                ):
                    new_symbols = await self.scanners[i].run(self.now)
                    if new_symbols:
                        really_new = [
                            x for x in new_symbols if x not in self.symbols
                        ]
                        if len(really_new) > 0:
                            print(
                                f"Loading data for {len(really_new)} symbols: {really_new}"
                            )
                            rc_msg.append(
                                f"Loaded data for {len(really_new)} symbols: {really_new}"
                            )
                            self.minute_history = {
                                **self.minute_history,
                                **(
                                    market_data.get_historical_data_from_poylgon_for_symbols(
                                        self.data_api,
                                        really_new,
                                        self.start - timedelta(days=7),
                                        self.start + timedelta(days=1),
                                    )
                                ),
                            }
                            self.symbols += really_new
                            print(f"loaded data for {len(really_new)} stocks")

            for symbol in self.symbols:
                try:
                    for strategy in trading_data.strategies:

                        try:
                            minute_index = self.minute_history[symbol][
                                "close"
                            ].index.get_loc(self.now, method="nearest")
                        except Exception as e:
                            print(f"[Exception] {self.now} {symbol} {e}")
                            print(self.minute_history[symbol]["close"][-100:])
                            continue

                        price = self.minute_history[symbol]["close"][
                            minute_index
                        ]

                        if symbol not in trading_data.positions:
                            trading_data.positions[symbol] = 0

                        do, what = await strategy.run(
                            symbol,
                            True,
                            int(trading_data.positions[symbol]),
                            self.minute_history[symbol][: minute_index + 1],
                            self.now,
                            self.portfolio_value,
                            debug=False,  # type: ignore
                            backtesting=True,
                        )
                        if do:
                            if (
                                what["side"] == "buy"
                                and float(what["qty"]) > 0
                                or what["side"] == "sell"
                                and float(what["qty"]) < 0
                            ):
                                trading_data.positions[symbol] += int(
                                    float(what["qty"])
                                )
                                trading_data.buy_time[
                                    symbol
                                ] = self.now.replace(second=0, microsecond=0)
                            else:
                                trading_data.positions[symbol] -= int(
                                    float(what["qty"])
                                )

                            trading_data.last_used_strategy[symbol] = strategy

                            rc_msg.append(
                                f"[{self.now}][{strategy.name}] {what['side']} {what['qty']} of {symbol} @ {price}"
                            )
                            db_trade = NewTrade(
                                algo_run_id=strategy.algo_run.run_id,
                                symbol=symbol,
                                qty=int(float(what["qty"])),
                                operation=what["side"],
                                price=price,
                                indicators=trading_data.buy_indicators[symbol]
                                if what["side"] == "buy"
                                else trading_data.sell_indicators[symbol],
                            )

                            await db_trade.save(
                                config.db_conn_pool,
                                str(self.now.to_pydatetime()),
                                trading_data.stop_prices[symbol],
                                trading_data.target_prices[symbol],
                            )

                            if what["side"] == "buy":
                                await strategy.buy_callback(
                                    symbol, price, int(float(what["qty"]))
                                )
                                break
                            elif what["side"] == "sell":
                                await strategy.sell_callback(
                                    symbol, price, int(float(what["qty"]))
                                )
                                break
                except Exception as e:
                    print(f"[Exception] {self.now} {symbol} {e}")
                    traceback.print_exc()

            self.now += timedelta(minutes=1)

            return True, rc_msg
        else:
            return False, []
Example #7
0
        async def backtest_symbol(
            symbol: str, scanner_start_time: datetime
        ) -> None:
            est = pytz.timezone("America/New_York")
            scanner_start_time = (
                pytz.utc.localize(scanner_start_time).astimezone(est)
                if scanner_start_time.tzinfo is None
                else scanner_start_time
            )
            start_time = pytz.utc.localize(start).astimezone(est)

            if scanner_start_time > start_time + duration:
                print(
                    f"{symbol} picked too late at {scanner_start_time} ({start_time}, {duration})"
                )
                return

            start_time = scanner_start_time
            if start_time.second > 0:
                start_time = start_time.replace(second=0, microsecond=0)
            print(
                f"--> back-testing {symbol} from {str(start_time)} duration {duration}"
            )
            if debug_symbols and symbol in debug_symbols:
                print("--> using DEBUG mode")

            re_try = 3

            while re_try > 0:
                # load historical data
                try:
                    symbol_data = data_api.polygon.historic_agg_v2(
                        symbol,
                        1,
                        "minute",
                        _from=str(start_time - timedelta(days=8)),
                        to=str(start_time + timedelta(days=1)),
                        limit=10000,
                    ).df
                except HTTPError as e:
                    tlog(f"Received HTTP error {e} for {symbol}")
                    return

                if len(symbol_data) < 100:
                    tlog(f"not enough data-points  for {symbol}")
                    return

                add_daily_vwap(
                    symbol_data,
                    debug=debug_symbols and symbol in debug_symbols,
                )
                market_data.minute_history[symbol] = symbol_data
                print(
                    f"loaded {len(market_data.minute_history[symbol].index)} agg data points"
                )

                position: int = 0
                try:
                    minute_index = symbol_data["close"].index.get_loc(
                        start_time, method="nearest"
                    )
                    break
                except (Exception, ValueError) as e:
                    print(f"[EXCEPTION] {e} - trying to reload-data. ")
                    re_try -= 1

            new_now = symbol_data.index[minute_index]
            print(f"start time with data {new_now}")
            price = 0.0
            last_run_id = None
            # start_time + duration
            while (
                new_now < config.market_close
                and minute_index < symbol_data.index.size - 1
            ):
                if symbol_data.index[minute_index] != new_now:
                    print(
                        "mismatch!", symbol_data.index[minute_index], new_now
                    )
                    print(
                        symbol_data["close"][
                            minute_index - 10 : minute_index + 1
                        ]
                    )
                    raise Exception()

                price = symbol_data["close"][minute_index]
                for strategy in trading_data.strategies:
                    if debug_symbols and symbol in debug_symbols:
                        print(
                            f"Execute strategy {strategy.name} on {symbol} at {new_now}"
                        )
                    do, what = await strategy.run(
                        symbol,
                        True,
                        position,
                        symbol_data[: minute_index + 1],
                        new_now,
                        portfolio_value,
                        debug=debug_symbols and symbol in debug_symbols,  # type: ignore
                        backtesting=True,
                    )
                    if do:
                        if (
                            what["side"] == "buy"
                            and float(what["qty"]) > 0
                            or what["side"] == "sell"
                            and float(what["qty"]) < 0
                        ):
                            position += int(float(what["qty"]))
                            trading_data.buy_time[symbol] = new_now.replace(
                                second=0, microsecond=0
                            )
                        else:
                            position -= int(float(what["qty"]))

                        trading_data.last_used_strategy[symbol] = strategy

                        db_trade = NewTrade(
                            algo_run_id=strategy.algo_run.run_id,
                            symbol=symbol,
                            qty=int(float(what["qty"])),
                            operation=what["side"],
                            price=price,
                            indicators=trading_data.buy_indicators[symbol]
                            if what["side"] == "buy"
                            else trading_data.sell_indicators[symbol],
                        )

                        await db_trade.save(
                            config.db_conn_pool,
                            str(new_now),
                            trading_data.stop_prices[symbol],
                            trading_data.target_prices[symbol],
                        )

                        if what["side"] == "buy":
                            await strategy.buy_callback(
                                symbol, price, int(float(what["qty"]))
                            )
                            break
                        elif what["side"] == "sell":
                            await strategy.sell_callback(
                                symbol, price, int(float(what["qty"]))
                            )
                            break
                    last_run_id = strategy.algo_run.run_id

                minute_index += 1
                new_now = symbol_data.index[minute_index]

            if position:
                if (
                    trading_data.last_used_strategy[symbol].type
                    == StrategyType.DAY_TRADE
                ):
                    tlog(
                        f"[{new_now}]{symbol} liquidate {position} at {price}"
                    )
                    db_trade = NewTrade(
                        algo_run_id=last_run_id,  # type: ignore
                        symbol=symbol,
                        qty=int(position)
                        if int(position) > 0
                        else -int(position),
                        operation="sell" if position > 0 else "buy",
                        price=price,
                        indicators={"liquidate": 1},
                    )
                    await db_trade.save(
                        config.db_conn_pool,
                        str(symbol_data.index[minute_index - 1]),
                    )
async def backtest_symbol(
    data_loader: DataLoader,
    portfolio_value: float,
    symbol: str,
    start: datetime,
    duration: timedelta,
    scanner_start_time: datetime,
    debug_symbol: bool = False,
) -> None:
    est = pytz.timezone("America/New_York")
    scanner_start_time = (pytz.utc.localize(scanner_start_time).astimezone(est)
                          if scanner_start_time.tzinfo is None else
                          scanner_start_time)
    start_time = pytz.utc.localize(start).astimezone(est)

    if scanner_start_time > start_time + duration:
        print(
            f"{symbol} picked too late at {scanner_start_time} ({start_time}, {duration})"
        )
        return

    start_time = scanner_start_time
    if start_time.second > 0:
        start_time = start_time.replace(second=0, microsecond=0)
    print(
        f"--> back-testing {symbol} from {str(start_time)} duration {duration}"
    )
    if debug_symbol:
        print("--> using DEBUG mode")

    symbol_data = pd.DataFrame(data_loader[symbol][start_time:start_time +
                                                   duration]  # type: ignore
                               )
    add_daily_vwap(
        symbol_data,
        debug=debug_symbol,
    )
    print(
        f"loaded {len(symbol_data)} agg data points({start_time}-{start_time + duration})"
    )

    minute_index = symbol_data["close"].index.get_loc(start_time,
                                                      method="nearest")

    position: int = 0
    new_now = symbol_data.index[minute_index]
    print(f"start time with data {new_now}")
    price = 0.0
    last_run_id = None

    # start_time + duration
    rejected: Dict[str, List] = {}
    while (new_now < config.market_close
           and minute_index < symbol_data.index.size - 1):
        if symbol_data.index[minute_index] != new_now:
            print("mismatch!", symbol_data.index[minute_index], new_now)
            print(symbol_data["close"][minute_index - 10:minute_index + 1])
            raise Exception()

        price = symbol_data["close"][minute_index]
        for strategy in trading_data.strategies:
            if debug_symbol:
                print(
                    f"Execute strategy {strategy.name} on {symbol} at {new_now}"
                )
            if symbol in rejected.get(strategy.name, []):
                continue

            try:
                do, what = await strategy.run(
                    symbol,
                    True,
                    position,
                    symbol_data[:minute_index + 1],
                    new_now,
                    portfolio_value,
                    debug=debug_symbol,  # type: ignore
                    backtesting=True,
                )
            except Exception as e:
                traceback.print_exc()
                tlog(
                    f"[ERROR] exception {e} on symbol {symbol} @ {strategy.name}"
                )
                continue

            if do:
                if (what["side"] == "buy" and float(what["qty"]) > 0
                        or what["side"] == "sell" and float(what["qty"]) < 0):
                    position += int(float(what["qty"]))
                    trading_data.buy_time[symbol] = new_now.replace(
                        second=0, microsecond=0)
                else:
                    position -= int(float(what["qty"]))

                trading_data.last_used_strategy[symbol] = strategy

                db_trade = NewTrade(
                    algo_run_id=strategy.algo_run.run_id,
                    symbol=symbol,
                    qty=int(float(what["qty"])),
                    operation=what["side"],
                    price=price,
                    indicators=trading_data.buy_indicators[symbol]
                    if what["side"] == "buy" else
                    trading_data.sell_indicators[symbol],
                )

                await db_trade.save(
                    config.db_conn_pool,
                    str(new_now),
                    trading_data.stop_prices[symbol],
                    trading_data.target_prices[symbol],
                )

                if what["side"] == "buy":
                    await strategy.buy_callback(symbol, price,
                                                int(float(what["qty"])))
                    break
                elif what["side"] == "sell":
                    await strategy.sell_callback(symbol, price,
                                                 int(float(what["qty"])))
                    break
            elif what.get("reject", False):
                if strategy.name in rejected:
                    rejected[strategy.name].append(symbol)
                else:
                    rejected[strategy.name] = [symbol]

            last_run_id = strategy.algo_run.run_id

        minute_index += 1
        new_now = symbol_data.index[minute_index]

    if position:
        if (trading_data.last_used_strategy[symbol].type ==
                StrategyType.DAY_TRADE):
            tlog(f"[{new_now}]{symbol} liquidate {position} at {price}")
            db_trade = NewTrade(
                algo_run_id=last_run_id,  # type: ignore
                symbol=symbol,
                qty=int(position) if int(position) > 0 else -int(position),
                operation="sell" if position > 0 else "buy",
                price=price,
                indicators={"liquidate": 1},
            )
            await db_trade.save(
                config.db_conn_pool,
                str(symbol_data.index[minute_index - 1]),
            )