コード例 #1
0
ファイル: watcher.py プロジェクト: Johannesduvenage/siis
class IGWatcher(Watcher):
    """
    IG watcher get price and volumes of instruments in live mode throught websocket API.

    Search Markets: https://demo-api.ig.com/gateway/deal/markets?searchTerm=USDJPY

    https://labs.ig.com/sample-apps/api-companion/index.html
    https://labs.ig.com/sample-apps/streaming-companion/index.html

    - LTV Last traded volume

    >> https://labs.ig.com/streaming-api-reference

    Limits
    ======

    Rest API:
        - Per-app non-trading requests per minute: 60
        - Per-account trading requests per minute: 100 (Applies to create/amend position or working order requests)
        - Per-account non-trading requests per minute: 30
        - Historical price data points per week: 10,000 (Applies to price history endpoints)

    Streaming API:
        - 40 concurrents subscriptions

    Data history:
        - 1 Sec 4 Days
        - 1 to 3 Min 40 Days
        - 5 Min to 4 Hours 360 Days
        - Day 15 years
        - Only for forex, indices, commodities, but no history for stocks !

    @todo get vol24 in base and quote unit
    @todo base_exchange_rate must be updated as price changes
    """

    MAX_CONCURRENT_SUBSCRIPTIONS = 40

    def __init__(self, service):
        super().__init__("ig.com", service, Watcher.WATCHER_PRICE_AND_VOLUME)

        self._host = "ig.com"
        self._connector = None
        self._lightstreamer = None
        self._subscriptions = []
        self._account_id = ""

        self._subscribed_markets = {}
        self._subscribed_ticks = {}

        self.__configured_symbols = set()  # cache for configured symbols set
        self.__matching_symbols = set()    # cache for matching symbols

        self._cached_tick = {}    # caches for when a value is not defined
        self._store_trade = True  # default store trade because we can't get history else

    def connect(self):
        super().connect()

        with self._mutex:
            try:
                self._ready = False
                self._connecting = True

                identity = self.service.identity(self._name)
                self._subscriptions = []  # reset previous list

                if identity:
                    self._host = identity.get('host')
                    self._account_type = "LIVE" if self._host == "api.ig.com" else "demo"
                    self._account_id = identity.get('account-id')

                    self._connector = IGConnector(
                        self.service,
                        identity.get('username'),
                        identity.get('password'),
                        identity.get('account-id'),
                        identity.get('api-key'),
                        identity.get('host'))

                    self._connector.connect()

                    # from CST and XST
                    password = "******" % (self._connector.cst, self._connector.xst)
                    # logger.debug(self._connector.cst, self._connector.xst, self._connector.lightstreamer_endpoint, identity.get('account-id'), self._connector.client_id)

                    if self._lightstreamer:
                        # destroy previous connection
                        self._lightstreamer.destroy()

                    self._lightstreamer = LSClient(
                        self._connector.lightstreamer_endpoint,  # "https://push.lightstreamer.com",
                        adapter_set="DEFAULT",
                        user=self._connector.client_id,
                        password=password)

                    self._lightstreamer.connect()

                    # subscribe for account and trades to have a reactive feedback and don't saturate the REST API
                    self.subscribe_account(identity.get('account-id'))
                    self.subscribe_trades(identity.get('account-id'))

                    #
                    # default watched instruments
                    #

                    configured_instruments = self.configured_symbols()

                    # @todo could check with API if configured epic exists and put them into this list
                    instruments = copy.copy(configured_instruments)

                    self._available_instruments = copy.copy(instruments)

                    configured_symbols = self.configured_symbols()
                    matching_symbols = self.matching_symbols_set(configured_symbols, instruments)

                    # cache them
                    self.__configured_symbols = configured_symbols
                    self.__matching_symbols = matching_symbols

                self._ready = True
                self._connecting = False

            except Exception as e:
                error_logger.error(repr(e))
                traceback_logger.error(traceback.format_exc())

                self._connector = None
                self._lightstreamer = None

        if self._connector and self._connector.connected and self._ready:
            self.service.notify(Signal.SIGNAL_WATCHER_CONNECTED, self.name, time.time())

    @property
    def connector(self):
        return self._connector

    @property
    def connected(self):
        return self._ready and self._connector is not None and self._connector.connected

    def disconnect(self):
        super().disconnect()

        with self._mutex:
            try:
                if self._lightstreamer:
                    # if self._lightstreamer.connected:
                    #   for sub_key in self._subscriptions:
                    #       self._lightstreamer.unsubscribe(sub_key)

                    self._subscriptions = []
                    self._lightstreamer.disconnect()
                    # self._lightstreamer._join()
                    self._lightstreamer = None

                if self._connector:
                    self._connector.disconnect()
                    self._connector = None

                    # reset subscribed markets WS
                    self._subscribed_markets = {}
                    self._subscribed_ticks = {}

                self._ready = False

            except Exception as e:
                error_logger.error(repr(e))
                traceback_logger.error(traceback.format_exc())

    def pre_update(self):
        if not self._connecting and not self._ready:
            reconnect = False

            with self._mutex:
                if self._connector is None or not self._connector.connected or self._lightstreamer is None or not self._lightstreamer.connected:
                    # cleanup
                    self._ready = False
                    self._connector = None

                    reconnect = True

            if reconnect:
                time.sleep(2)
                self.connect()
                return

    def update(self):
        if not super().update():
            return False

        if not self.connected:
            return False

        #
        # ohlc close/open
        #

        with self._mutex:
            self.update_from_tick()

        #
        # market info update (each 4h)
        #

        if time.time() - self._last_market_update >= IGWatcher.UPDATE_MARKET_INFO_DELAY:  # only once per 4h
            try:
                # session must at least be obtained each 6h, we call each 4h at least but if we have a server invalidation
                self.update_markets_info()
                self._last_market_update = time.time()
            except Exception as e:
                error_logger.error(repr(e))
                traceback_logger.error(traceback.format_exc())

                # @todo and is the WS still valid ?
                self._connector.update_session()

        return True

    def post_update(self):
        super().post_update()
        time.sleep(0.0005)

    def post_run(self):
        super().post_run()

    #
    # instruments
    #

    def subscribe(self, market_id, timeframe, ohlc_depths=None, order_book_depth=None):
        with self._mutex:
            if market_id in self.__matching_symbols:
                # fetch from 1m to 1w, we have a problem of the 10k candle limit per weekend, then we only
                # prefetch for the last of each except for 1m and 5m we assume we have a delay of 5 minutes
                # from the manual prefetch script execution and assuming the higher timeframe are already up-to-date.
                if self._initial_fetch:
                    logger.info("%s prefetch for %s" % (self.name, market_id))

                    self.fetch_and_generate(market_id, Instrument.TF_1M, 5, None)
                    self.fetch_and_generate(market_id, Instrument.TF_3M, 2, None)
                    self.fetch_and_generate(market_id, Instrument.TF_5M, 1, None)
                    self.fetch_and_generate(market_id, Instrument.TF_15M, 1, None)
                    self.fetch_and_generate(market_id, Instrument.TF_1H, 1, None)
                    self.fetch_and_generate(market_id, Instrument.TF_4H, 1, None)
                    self.fetch_and_generate(market_id, Instrument.TF_1D, 1, None)
                    self.fetch_and_generate(market_id, Instrument.TF_1W, 1, None)

                self.insert_watched_instrument(market_id, [0])

                # to know when market close but could be an hourly REST API call, but it consume one subscriber...
                # @todo so maybe prefers REST call hourly ? but need bid/ofr properly defined at signals on trader.market and strategy.instrument !
                self.subscribe_market(market_id)

                # tick data
                self.subscribe_tick(market_id)

                # if self._initial_fetch:
                #     logger.info("Watcher %s wait 8+1 seconds to limit to a fair API usage" % (self.name,))
                #     time.sleep(9.0)  # 1 sec per query + 1 extra second

                return True

        return False

    def unsubscribe(self, market_id, timeframe):
        with self._mutex:
            if market_id in self._subscribed_markets:
                sub = self._subscribed_markets[market_id]
                self.unsubscribe_ws(sub)
                del self._subscribed_markets[market_id]

                sub = self._subscribed_ticks[market_id]
                self.unsubscribe_ws(sub)
                del self._subscribed_ticks[market_id]

                return True

        return False

    #
    # WS subscribtion
    #

    def subscribe_account(self, account_id):
        fields = ["PNL", "AVAILABLE_TO_DEAL", "MARGIN", "FUNDS", "AVAILABLE_CASH"]

        subscription = Subscription(
                mode="MERGE",
                items=["ACCOUNT:"+account_id],
                fields=fields,
                adapter="")

        self.subscribe_ws(subscription)
        subscription.addlistener(self, IGWatcher.on_account_update)

    def subscribe_trades(self, account_id):
        fields = ["CONFIRMS", "OPU", "WOU"]

        subscription = Subscription(
                mode="DISTINCT",
                items=["TRADE:"+account_id],
                fields=fields,
                adapter="")

        self.subscribe_ws(subscription)
        subscription.addlistener(self, IGWatcher.on_trade_update)

    def subscribe_tick(self, instrument):
        """
        Subscribe to an instrument tick updates.
        """
        fields = ["BID", "OFR", "LTP", "LTV", "TTV", "UTM"]

        subscription = Subscription(
            mode="DISTINCT",
            items=["CHART:"+instrument+":TICK"],
            fields=fields,
            adapter="")

        sub_key = self.subscribe_ws(subscription)
        subscription.addlistener(self, IGWatcher.on_tick_update)

        self._subscribed_ticks[instrument] = sub_key

    # def subscribe_ohlc(self, instrument, timeframe):
    #     """
    #     Subscribe to an instrument. Timeframe must be greater than 0.
    #     """
    #     fields = [
    #         "BID_OPEN", "OFR_OPEN",
    #         "BID_CLOSE", "OFR_CLOSE",
    #         "BID_HIGH", "OFR_HIGH",
    #         "BID_LOW", "OFR_LOW",
    #         "LTP", "LTV", "TTV", "UTM",
    #         "CONS_END"
    #     ]

    #     if timeframe == Instrument.TF_SEC:
    #         tf = "SECOND"
    #     elif timeframe == Instrument.TF_MIN:
    #         tf = "1MINUTE"
    #     elif timeframe == Instrument.TF_5MIN:
    #         tf = "5MINUTE"
    #     elif timeframe == Instrument.TF_HOUR:
    #         tf = "HOUR"
    #     else:
    #         return

    #     subscription = Subscription(
    #         mode="MERGE",
    #         items=["CHART:"+instrument+":"+tf],
    #         fields=fields,
    #         adapter="")

    #     self.subscribe_ws(subscription)
    #     subscription.addlistener(self, IGWatcher.on_ohlc_update)

    def subscribe_market(self, instrument):
        """
        Subscribe to an instrument.
        """
        fields = ["MARKET_STATE", "UPDATE_TIME", "BID", "OFFER"]

        subscription = Subscription(
            mode="MERGE",
            items=["MARKET:"+instrument],
            fields=fields,
            adapter="")

        sub_key = self.subscribe_ws(subscription)
        subscription.addlistener(self, IGWatcher.on_market_update)

        self._subscribed_markets[instrument] = sub_key

    def subscribe_ws(self, subscription):
        """
        Registering the Subscription
        """
        sub_key = self._lightstreamer.subscribe(subscription)
        self._subscriptions.append(sub_key)

        return sub_key

    def unsubscribe_ws(self, sub_key):
        if sub_key in self._subscriptions:
            self._lightstreamer.unsubscribe(sub_key)
            del self._subscriptions[sub_key]

    #
    # WS data
    #

    @staticmethod
    def on_account_update(self, item_update):
        name = item_update.get('name', '').split(':')

        try:
            if len(name) == 2 and name[0] == 'ACCOUNT' and name[1] == self._account_id:
                # live account updates
                values = item_update['values']

                account_data = (float(values['FUNDS']), float(values['AVAILABLE_TO_DEAL']), float(values['PNL']), None, float(values['MARGIN']))
                self.service.notify(Signal.SIGNAL_ACCOUNT_DATA, self.name, account_data)
        except Exception as e:
            error.error(repr(e))
            traceback_logger.error(traceback.format_exc())

    @staticmethod
    def on_market_update(self, item_update):
        name = item_update.get('name', '').split(':')

        try:
            if len(name) == 2 and name[0] == 'MARKET':
                # market data instrument by epic
                values = item_update['values']
                epic = name[1]

                ready = values['MARKET_STATE'] == 'TRADEABLE'

                # date of the event 20:36:01 without Z
                if ready:
                    # @todo take now and replace H:M:S
                    update_time = time.time()  #  datetime.strptime(values['UPDATE_TIME'], '%H:%M:%S').timestamp()
                    market_data = (name[1], True, update_time, float(values["BID"]), float(values["OFFER"]), None, None, None, None, None)
                else:
                    update_time = 0
                    market_data = (name[1], False, 0, None, None, None, None, None, None, None)

                self.service.notify(Signal.SIGNAL_MARKET_DATA, self.name, market_data)
        except Exception as e:
            logger.debug(repr(e))
            error_logger.error(traceback.format_exc())

    @staticmethod
    def on_tick_update(self, item_update):
        name = item_update.get('name', '').split(':')

        try:
            if len(name) == 3 and name[0] == 'CHART' and name[2] == 'TICK':
                values = item_update['values']
                market_id = name[1]

                bid = None
                ofr = None
                utm = None
                ltv = None

                if values['UTM']:
                    utm = values['UTM']
                elif market_id in self._cached_tick:
                    utm = self._cached_tick[market_id][0]

                if values['BID']:
                    bid = values['BID']
                elif market_id in self._cached_tick:
                    bid = self._cached_tick[market_id][1]

                if values['OFR']:
                    ofr = values['OFR']
                elif market_id in self._cached_tick:
                    ofr = self._cached_tick[market_id][2]

                if values['LTV']:
                    ltv = values['LTV']
                elif market_id in self._cached_tick:
                    ltv = self._cached_tick[market_id][3]

                if utm is None or bid is None or ofr is None:
                    # need all informations, wait the next one
                    return

                # cache for when a value is not defined
                self._cached_tick[market_id] = (utm, bid, ofr, ltv)

                tick = (float(utm) * 0.001, float(bid), float(ofr), float(ltv or "0"))

                self.service.notify(Signal.SIGNAL_TICK_DATA, self.name, (market_id, tick))

                for tf in Watcher.STORED_TIMEFRAMES:
                    # generate candle per each tf
                    with self._mutex:
                        candle = self.update_ohlc(market_id, tf, tick[0], tick[1], tick[2], tick[3])

                    if candle is not None:
                        self.service.notify(Signal.SIGNAL_CANDLE_DATA, self.name, (market_id, candle))

                # disabled for now
                if not self._read_only and self._store_trade:
                    Database.inst().store_market_trade((self.name, market_id, int(utm), bid, ofr, ltv or 0))

        except Exception as e:
            error_logger.error(repr(e))
            traceback_logger.error(traceback.format_exc())

    # @staticmethod
    # def on_ohlc_update(self, item_update):
    #     name = item_update.get('name', '').split(':')

    #     try:
    #         if len(name) == 3 and name[0] == 'CHART':
    #             values = item_update['values']
    #             if values['CONS_END'] == '0':
    #                 # get only consolidated candles
    #                 # @warning It is rarely defined, so many close could be missing, prefers using tick to rebuild ohlc locally
    #                 return

    #             # timeframe
    #             if name[2] == 'SECOND':
    #                 tf = Instrument.TF_SEC
    #             elif name[2] == '1MINUTE':
    #                 tf = Instrument.TF_MIN
    #             elif name[2] == '5MINUTE':
    #                 tf = Instrument.TF_5MIN
    #             elif name[2] == 'HOUR':
    #                 tf = Instrument.TF_HOUR
 
    #             # one of the value could be missing, use the previous from the cache if we have it
    #             if (values['UTM'] is None or values['LTV'] is None or
    #                 values['OFR_OPEN'] is None or values['OFR_HIGH'] is None or values['OFR_LOW'] is None or values['OFR_CLOSE'] is None or 
    #                     values['BID_OPEN'] is None or values['BID_HIGH'] is None or values['BID_LOW'] is None or values['BID_CLOSE'] is None):

    #                 if name[1] not in self._cached_ohlc or tf not in self._cached_ohlc[name[1]]:
    #                     logger.warning("no value and cache miss for %s ohlc in %s (%s)" % (name[1], tf, values))

    #                 if values['UTM'] is None:
    #                     utm = self._cached_ohlc[name[1]][tf][0]
    #                 if values['LTV'] is None:
    #                     ltv = self._cached_ohlc[name[1]][tf][9]

    #             utm = values['UTM']
    #             ltv = values['LTV']

    #             candle = Candle(float(utm) * 0.001, tf)

    #             # if incomplete candle replace ofr by bid or bid by ofr @todo but must be value from previous candle
    #             # but and if we don't have to previous... ok for 1 min but for 1h ? ...
    #             bid_open = values['BID_OPEN'] or values['OFR_OPEN']
    #             bid_high = values['BID_HIGH'] or values['OFR_HIGH']
    #             bid_low = values['BID_LOW'] or values['OFR_LOW']
    #             bid_close = values['BID_CLOSE'] or values['OFR_CLOSE']

    #             ofr_open = values['OFR_OPEN'] or values['BID_OPEN']
    #             ofr_high = values['OFR_HIGH'] or values['BID_HIGH']
    #             ofr_low = values['OFR_LOW'] or values['BID_LOW']
    #             ofr_close = values['OFR_CLOSE'] or values['BID_CLOSE']

    #             candle.set_bid_ohlc(float(bid_open), float(bid_high), float(bid_low), float(bid_close))
    #             candle.set_ofr_ohlc(float(ofr_open), float(ofr_high), float(ofr_low), float(ofr_close))
    #             candle.set_volume(float(values['LTV']) if values['LTV'] else 0.0)
    #             candle.set_consolidated(values['CONS_END'] == '1')

    #             self.service.notify(Signal.SIGNAL_CANDLE_DATA, self.name, (name[1], candle))

    #             if values['CONS_END'] == '1' and not self._read_only:
    #                 # write only consolidated candles. values are string its perfect if not last traded volume then 0
    #                 Database.inst().store_market_ohlc((
    #                     self.name, name[1], int(utm), tf,
    #                     bid_open, bid_high, bid_low, bid_close,
    #                     ofr_open, ofr_high, ofr_low, ofr_close,
    #                     values['LTV'] or "0"))

    #             # cache for when a value is not defined
    #             self._cached_ohlc[name[1]][tf] = (utm, bid_open, bid_high, bid_low, bid_close, ofr_open, ofr_high, ofr_low, ofr_close, ltv)

    #     except Exception as e:
    #         error_logger.error(repr(e))
    #         traceback_logger.error(traceback.format_exc())

    @staticmethod
    def on_trade_update(self, item_update):
        name = item_update.get('name', '').split(':')

        try:
            if len(name) == 2 and name[0] == 'TRADE' and name[1] == self._account_id:
                # live trade updates
                values = item_update['values']

                #
                # active waiting order (open/updated/deleted)
                #

                if values.get('WOU'):
                    data = json.loads(values.get('WOU'))
                    exec_logger.info("ig.com WOU %s" % str(data))

                    order_id = data['dealId']
                    ref_order_id = data['dealReference']

                    epic = data['epic']

                    # date of the event 2018-09-13T20:36:01.096 without Z
                    event_time = datetime.strptime(data['timestamp'], '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=UTC()).timestamp()

                    if data.get('direction', '') == 'BUY':
                        direction = Order.LONG
                    elif data.get('direction', '') == 'SELL':
                        direction = Order.SHORT
                    else:
                        direction = 0

                    if data.get('dealStatus', "") == 'REJECTED':
                        pass
                    elif data.get('dealStatus', "") == 'ACCEPTED':
                        quantity = float(data.get('size')) if data.get('size') is not None else 0.0
                        level = float(data['level']) if data.get('level') is not None else None
                        stop_distance = float(data['stopDistance']) if data.get('stopDistance') is not None else None
                        limit_distance = float(data['limitDistance']) if data.get('limitDistance') is not None else None
                        guaranteed_stop = data.get('guaranteedStop', False)
                        currency = data.get('currency', "")

                        if data.get('orderType'):
                            if data['orderType'] == "LIMIT":
                                order_type = Order.ORDER_LIMIT
                            elif data['orderType'] == "STOP":
                                order_type = Order.ORDER_STOP
                            else:
                                order_type = Order.ORDER_MARKET
                        else:
                            order_type = Order.ORDER_MARKET

                        if data.get('timeInForce'):
                            if data['timeInForce'] == "GOOD_TILL_CANCELLED":
                                time_in_force = Order.TIME_IN_FORCE_GTC
                            elif data['timeInForce'] == "GOOD_TILL_DATE":
                                time_in_force = Order.TIME_IN_FORCE_GTD
                                # data['goodTillDate']   @todo till date
                        else:
                            time_in_force = Order.TIME_IN_FORCE_GTC

                        status = data.get('status', "")

                        if status == "OPEN":
                            order_data = {
                                'id': order_id,
                                'type': order_type,
                                'time-in-force': time_in_force,
                                'price': level if order_type == Order.ORDER_LIMIT else None,
                                'stop-price': level if order_type == Order.ORDER_STOP else None,
                                'stop-loss': stop_distance,
                                'take-profit': limit_distance
                            }

                            self.service.notify(Signal.SIGNAL_ORDER_OPENED, self.name, (epic, order_data, ref_order_id))

                        elif status == "UPDATED":
                            # signal of updated order
                            order_data = {
                                'id': order_id,
                                'type': order_type,
                                'time-in-force': time_in_force,
                                'price': level if order_type == Order.ORDER_LIMIT else None,
                                'stop-price': level if order_type == Order.ORDER_STOP else None,
                                'stop-loss': stop_distance,
                                'take-profit': limit_distance
                            }

                            self.service.notify(Signal.SIGNAL_ORDER_UPDATED, self.name, (epic, order_data, ref_order_id))

                        elif status == "DELETED":
                            # signal of deleted order
                            self.service.notify(Signal.SIGNAL_ORDER_DELETED, self.name, (epic, order_id, ref_order_id))

                #
                # order confirms (accepted/rejected)
                #

                if values.get('CONFIRMS'):
                    data = json.loads(values.get('CONFIRMS'))
                    exec_logger.info("ig.com CONFIRMS %s" % str(data))

                    epic = data.get('epic')

                    if data.get('dealStatus', "") == "REJECTED":
                        ref_order_id = data['dealReference']

                        self.service.notify(Signal.SIGNAL_ORDER_REJECTED, self.name, (epic, ref_order_id))

                    elif data.get('dealStatus', "") == "ACCEPTED":
                        # deal confirmed and accepted
                        order_id = data['dealId']
                        ref_order_id = data['dealReference']

                        # date 2018-09-13T20:36:01.096 without Z
                        event_time = datetime.strptime(data['date'], '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=UTC()).timestamp()

                        # direction of the trade
                        if data['direction'] == 'BUY':
                            direction = Order.LONG
                        elif data['direction'] == 'SELL':
                            direction = Order.SHORT
                        else:
                            direction = 0

                        level = float(data['level']) if data.get('level') is not None else None   # exec price
                        quantity = float(data['size']) if data.get('size') is not None else 0.0
                        stop_level = float(data['stopLevel']) if data.get('stopLevel') is not None else 0.0
                        limit_level = float(data['limitLevel']) if data.get('limitLevel') is not None else 0.0
                        profit_loss = float(data['profit']) if data.get('profit') is not None else 0.0
                        profit_currency = data.get('profitCurrency', "")

                        # 'expiry', 'guaranteedStop'

                        # affected positions, normaly should not be necessary except if user create a manual trade that could reduce an existing position
                        # for affected_deal in data.get('affectedDeals', []):
                        #     position_id = affected_deal['dealId']
                        #     status = affected_deal.get('status', "")
                        #     if status == "AMENDED":
                        #         pass
                        #     elif status == "DELETED":
                        #         pass
                        #     elif status == "FULLY_CLOSED":
                        #         pass
                        #     elif status == "OPENED":
                        #         pass
                        #     elif status == "PARTIALLY_CLOSED":
                        #         pass

                        status = data.get('status', "")

                        if status == "AMENDED":
                            # traded and initial
                            order = {
                                'id': order_id,
                                'symbol': epic,
                                'timestamp': event_time,
                                'direction': direction,
                                'quantity': None,  # no have
                                'filled': None,  # no have
                                'cumulative-filled': quantity,
                                'exec-price': level,
                                'avg-price': None,
                                'stop-loss': stop_level,
                                'take-profit': limit_level,
                                'profit-loss': profit_loss,
                                'profit-currency': profit_currency,
                                'info': 'amended'
                            }

                            self.service.notify(Signal.SIGNAL_ORDER_TRADED, self.name, (epic, order, ref_order_id))

                        elif status == "CLOSED":
                            # traded and completed
                            order = {
                                'id': order_id,
                                'symbol': epic,
                                'timestamp': event_time,
                                'direction': direction,
                                'quantity': None,
                                'filled': None,
                                'cumulative-filled': quantity,
                                'exec-price': level,
                                'avg-price': None,
                                'profit-loss': profit_loss,
                                'profit-currency': profit_currency,
                                'info': 'closed'
                            }

                            if data.get('limitLevel') and data.get('stopLevel'):
                                order['type'] = Order.ORDER_STOP_LIMIT
                            elif data.get('limitLevel'):
                                order['type'] = Order.ORDER_LIMIT
                            elif data.get('stopLevel'):
                                order['type'] = Order.ORDER_STOP
                            else:
                                order['type'] = Order.ORDER_MARKET

                            self.service.notify(Signal.SIGNAL_ORDER_TRADED, self.name, (epic, order, ref_order_id))
                            self.service.notify(Signal.SIGNAL_ORDER_DELETED, self.name, (epic, order_id, ""))

                        elif status == "DELETED":
                            # deleted why for, we never receive them
                            self.service.notify(Signal.SIGNAL_ORDER_DELETED, self.name, (epic, order_id, ""))

                        elif status == "OPEN":
                            # traded and initial
                            order = {
                                'id': order_id,
                                'symbol': epic,
                                'timestamp': event_time,
                                'direction': direction,
                                'quantity': None,  # no have
                                'filled': None,  # no have
                                'cumulative-filled': quantity,
                                'exec-price': level,
                                'avg-price': None,
                                'stop-loss': stop_level,
                                'take-profit': limit_level,
                                'profit-loss': profit_loss,
                                'profit-currency': profit_currency,
                                'info': 'open'
                            }

                            if data.get('limitLevel') and data.get('stopLevel'):
                                order['type'] = Order.ORDER_STOP_LIMIT
                                order['price'] = float(data.get('limitLevel'))
                                order['stop-price'] = float(data.get('stopLevel'))
                            elif data.get('limitLevel'):
                                order['type'] = Order.ORDER_LIMIT
                                order['price'] = float(data.get('limitLevel'))
                            elif data.get('stopLevel'):
                                order['type'] = Order.ORDER_STOP
                                order['stop-price'] = float(data.get('stopLevel'))
                            else:
                                order['type'] = Order.ORDER_MARKET

                            # @todo 'limitDistance' 'stopDistance' 'trailingStop'

                            self.service.notify(Signal.SIGNAL_ORDER_OPENED, self.name, (epic, order, ref_order_id))

                            if quantity > 0.0:
                                self.service.notify(Signal.SIGNAL_ORDER_TRADED, self.name, (epic, order, ref_order_id))

                        elif status == "PARTIALLY_CLOSED":
                            # traded and partially completed
                            order = {
                                'id': order_id,
                                'symbol': epic,
                                'timestamp': event_time,
                                'direction': direction,
                                'quantity': None,  # no have
                                'filled': None,  # no have
                                'cumulative-filled': quantity,
                                'exec-price': level,
                                'avg-price': None,
                                'profit-loss': profit_loss,
                                'profit-currency': profit_currency,
                                'info': 'partially-closed'
                            }

                            self.service.notify(Signal.SIGNAL_ORDER_TRADED, self.name, (epic, order, ref_order_id))

                #
                # active position (open/updated/deleted)
                #

                if values.get('OPU'):
                    data = json.loads(values.get('OPU'))
                    exec_logger.info("ig.com OPU %s" % str(data))

                    position_id = data['dealId']
                    ref_order_id = data['dealReference']

                    epic = data.get('epic')
                    # "channel": "WTP", "expiry": "-"

                    # date of the event 2018-09-13T20:36:01.096 without Z
                    event_time = datetime.strptime(data['timestamp'], '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=UTC()).timestamp()

                    if data.get('direction', '') == 'BUY':
                        direction = Order.LONG
                    elif data.get('direction', '') == 'SELL':
                        direction = Order.SHORT
                    else:
                        direction = Order.LONG

                    if data.get('dealStatus', "") == "REJECTED":
                        pass

                    elif data.get('dealStatus', "") == "ACCEPTED":
                        quantity = float(data.get('size')) if data.get('size') is not None else 0.0
                        level = float(data['level']) if data.get('level') is not None else None
                        stop_level = float(data['stopLevel']) if data.get('stopLevel') is not None else None
                        limit_level = float(data['limitLevel']) if data.get('limitLevel') is not None else None
                        profit_loss = float(data['profit']) if data.get('profit') is not None else 0.0
                        profit_currency = data.get('profitCurrency', "")
                        # @todo trailingStep, trailingStopDistance, guaranteedStop

                        status = data.get('status', "")

                        if status == "OPEN":
                            # signal of opened position
                            position_data = {
                                'id': position_id,
                                'symbol': epic,
                                'direction': direction,
                                'timestamp': event_time,
                                'quantity': quantity,
                                'exec-price': level,
                                'avg-price': level,
                                'avg-entry-price': level,  # entry
                                'stop-loss': stop_level,
                                'take-profit': limit_level,
                                'profit-loss': profit_loss,
                                'profit-currency': profit_currency,
                                'cumulative-filled': None,
                                'filled': None,
                                'liquidation-price': None
                            }

                            self.service.notify(Signal.SIGNAL_POSITION_OPENED, self.name, (epic, position_data, ref_order_id))

                        elif status == "UPDATED":
                            # signal of updated position
                            position_data = {
                                'id': position_id,
                                'symbol': epic,
                                'direction': direction,
                                'timestamp': event_time,
                                'quantity': quantity,
                                'exec-price': level,
                                'avg-entry-price': level,  # entry
                                'avg-price': level,
                                'stop-loss': stop_level,
                                'take-profit': limit_level,
                                'profit-loss': profit_loss,
                                'profit-currency': profit_currency,
                                'cumulative-filled': None,
                                'filled': None,
                                'liquidation-price': None
                            }

                            self.service.notify(Signal.SIGNAL_POSITION_UPDATED, self.name, (epic, position_data, ref_order_id))

                        elif status == "DELETED":
                            # signal of deleted position
                            position_data = {
                                'id': position_id,
                                'symbol': epic,
                                'direction': direction,
                                'timestamp': event_time,
                                'quantity': quantity,
                                'exec-price': level,
                                'avg-price': level,
                                'avg-exit-price': level,  # exit
                                'stop-loss': stop_level,
                                'take-profit': limit_level,
                                'profit-loss': profit_loss,
                                'profit-currency': profit_currency,
                                'cumulative-filled': None,
                                'filled': None,
                                'liquidation-price': None
                            }
                            self.service.notify(Signal.SIGNAL_POSITION_DELETED, self.name, (epic, position_data, ref_order_id))

        except Exception as e:
            error_logger.error(repr(e))
            traceback_logger.error(traceback.format_exc())

    #
    # REST data
    #

    def fetch_market(self, epic):
        """
        Fetch and cache it. It rarely changes, except for base exchange rate, so assume it once for all.
        """
        market_info = self._connector.market(epic)

        instrument = market_info['instrument']
        snapshot = market_info['snapshot']
        dealing_rules = market_info['dealingRules']

        market = Market(epic, instrument['marketId'])

        # cannot interpret this value because IG want it as it is
        market.expiry = instrument['expiry']

        # not perfect but IG does not provides this information
        if instrument["marketId"].endswith(instrument["currencies"][0]["name"]):
            base_symbol = instrument["marketId"][:-len(instrument["currencies"][0]["name"])]
        else:
            base_symbol = instrument["marketId"]

        market.base_exchange_rate = instrument['currencies'][0]['baseExchangeRate']   # "exchangeRate": 0.77

        market.one_pip_means = float(instrument['onePipMeans'].split(' ')[0])  # "1 Index Point", "0.0001 USD/EUR"
        market.value_per_pip = float(instrument['valueOfOnePip'])
        market.contract_size = float(instrument['contractSize'])
        market.lot_size = float(instrument['lotSize'])

        market.set_base(
            base_symbol,
            base_symbol,
            decimal_place(market.one_pip_means))

        if instrument['type'] in ('CURRENCIES'):
            pass  # quote_precision = 
        elif instrument['type'] in ('INDICES', 'COMMODITIES', 'SHARES', 'RATES', 'SECTORS'):
            pass  # quote_precision = 

        market.set_quote(
            instrument["currencies"][0]["name"],
            instrument["currencies"][0]['symbol'],
            decimal_place(market.one_pip_means))

        # "forceOpenAllowed": true,
        # "stopsLimitsAllowed": true,
        # "controlledRiskAllowed": true,
        # "streamingPricesAvailable": true,

        if snapshot:
            market.is_open = snapshot["marketStatus"] == "TRADEABLE"
            market.bid = snapshot['bid']
            market.ofr = snapshot['offer']

        # "marginFactorUnit": "PERCENTAGE" not avalaible if market is down
        if instrument.get('marginFactor') and market.is_open:
            market.margin_factor = float(instrument['marginFactor'])
            margin_factor = instrument['marginFactor']
        elif instrument.get('margin') and market.is_open:
            market.margin_factor = 0.1 / float(instrument['margin'])
            margin_factor = str(market.margin_factor)
        else:
            # we don't want this when market is down because it could overwrite the previous stored value
            margin_factor = None

        if instrument['unit'] == 'AMOUNT':
            market.unit_type = Market.UNIT_AMOUNT
        elif instrument['unit'] == 'CONTRACTS':
            market.unit_type = Market.UNIT_CONTRACTS
        elif instrument['unit'] == 'SHARES':
            market.unit_type = Market.UNIT_SHARES

        # BINARY OPT_* BUNGEE_* 
        if instrument['type'] == 'CURRENCIES':
            market.market_type = Market.TYPE_CURRENCY
        elif instrument['type'] == 'INDICES':
            market.market_type = Market.TYPE_INDICE
        elif instrument['type'] == 'COMMODITIES':
            market.market_type = Market.TYPE_COMMODITY
        elif instrument['type'] == 'SHARES':
            market.market_type = Market.TYPE_STOCK
        elif instrument['type'] == 'RATES':
            market.market_type = Market.TYPE_RATE
        elif instrument['type'] == 'SECTORS':
            market.market_type = Market.TYPE_SECTOR

        market.trade = Market.TRADE_MARGIN | Market.TRADE_POSITION
        market.contract_type = Market.CONTRACT_CFD

        # take minDealSize as tick size
        market.set_size_limits(dealing_rules["minDealSize"]["value"], 0.0, dealing_rules["minDealSize"]["value"])
        # @todo there is some limits in contract size
        market.set_notional_limits(0.0, 0.0, 0.0)
        # use one pip means for minimum and tick price size
        market.set_price_limits(market.one_pip_means, 0.0, market.one_pip_means)

        # commission for stocks @todo
        commission = "0.0"

        # store the last market info to be used for backtesting
        if not self._read_only:
            Database.inst().store_market_info((self.name, epic, market.symbol,
                market.market_type, market.unit_type, market.contract_type,  # type
                market.trade, market.orders,  # type
                market.base, market.base_display, market.base_precision,  # base
                market.quote, market.quote_display, market.quote_precision,  # quote
                market.expiry, int(market.last_update_time * 1000.0),  # expiry, timestamp
                instrument['lotSize'], instrument['contractSize'], str(market.base_exchange_rate),
                instrument['valueOfOnePip'], instrument['onePipMeans'].split(' ')[0], margin_factor,
                dealing_rules["minDealSize"]["value"], "0.0", dealing_rules["minDealSize"]["value"],  # size limits
                "0.0", "0.0", "0.0",  # notional limits
                "0.0", "0.0", "0.0",  # price limits
                "0.0", "0.0", commission, commission)  # fees
            )

        # print(market.symbol, market._size_limits, market._price_limits)

        # notify for strategy
        self.service.notify(Signal.SIGNAL_MARKET_INFO_DATA, self.name, (epic, market))

        return market

    def update_markets_info(self):
        """
        Update market info (very important because IG frequently changes lot or contract size).
        """
        for market_id in self._watched_instruments:
            market = self.fetch_market(market_id)

            if market.is_open:
                market_data = (market_id, market.is_open, market.last_update_time, market.bid, market.ofr,
                        market.base_exchange_rate, market.contract_size, market.value_per_pip,
                        market.vol24h_base, market.vol24h_quote)
            else:
                market_data = (market_id, market.is_open, market.last_update_time, None, None, None, None, None, None, None)

            self.service.notify(Signal.SIGNAL_MARKET_DATA, self.name, market_data)

    def fetch_candles(self, market_id, timeframe, from_date=None, to_date=None, n_last=None):
        try:
            if n_last:
                data = self._connector.history_last_n(market_id, timeframe, n_last)
            else:
                data = self._connector.history_range(market_id, timeframe, from_date, to_date)
        except Exception as e:
            error_logger.error(repr(e))
            traceback_logger.error(traceback.format_exc())

            data = {}

        prices = data.get('prices', [])

        for price in prices:
            timestamp = datetime.strptime(price['snapshotTime'], '%Y:%m:%d-%H:%M:%S').timestamp()

            if price.get('highPrice')['bid'] is None and price.get('highPrice')['ask'] is None:
                # ignore empty candles
                continue

            # yield (timestamp, high bid, low, open, close, high ofr, low, open, close, volume)
            yield([int(timestamp * 1000),
                str(price.get('highPrice')['bid'] or price.get('highPrice')['ask']),
                str(price.get('lowPrice')['bid'] or price.get('lowPrice')['ask']),
                str(price.get('openPrice')['bid'] or price.get('openPrice')['ask']),
                str(price.get('closePrice')['bid'] or price.get('closePrice')['ask']),
                str(price.get('highPrice')['ask'] or price.get('highPrice')['bid']),
                str(price.get('lowPrice')['ask'] or price.get('lowPrice')['bid']),
                str(price.get('openPrice')['ask'] or price.get('openPrice')['bid']),
                str(price.get('closePrice')['ask'] or price.get('closePrice')['bid']),
                price.get('lastTradedVolume', '0')])
コード例 #2
0
class IGWatcher(Watcher):
    """
    IG watcher get price and volumes of instruments in live mode throught websocket API.

    Search Markets: https://demo-api.ig.com/gateway/deal/markets?searchTerm=USDJPY

    https://labs.ig.com/sample-apps/api-companion/index.html
    https://labs.ig.com/sample-apps/streaming-companion/index.html

    - LTV Last traded volume

    >> https://labs.ig.com/streaming-api-reference

    Limits
    ======

    Rest API:
        - Per-app non-trading requests per minute: 60
        - Per-account trading requests per minute: 100 (Applies to create/amend position or working order requests)
        - Per-account non-trading requests per minute: 30
        - Historical price data points per week: 10,000 (Applies to price history endpoints)

    Streaming API:
        - 40 concurrents subscriptions

    Data history:
        - 1 Sec 4 Days
        - 1 to 3 Min 40 Days
        - 5 Min to 4 Hours 360 Days
        - Day 15 years

    @todo get vol24 in base and quote unit
    @todo base_exchange_rate must be updated as price changes

    @todo could use endpoint marketnavigation to get all instruments but its hierarchically queries...
        { "nodes": [{ "id": "668394", "name": "Crypto-monnaie" }, { "id": "5371876", ...
        per nodes id we have then : {"nodes": [{ "id": "668997", "name": "Bitcoin" }, { "id": "1002200", ...
        and finally when we found "markets": [{ "epic": "CS.D.AUDUSD.CFD.IP", ... }]
    """

    MAX_CONCURRENT_SUBSCRIPTIONS = 40

    def __init__(self, service):
        super().__init__("ig.com", service, Watcher.WATCHER_PRICE_AND_VOLUME)

        self._host = "ig.com"
        self._connector = None
        self._lightstreamer = None
        self._subscriptions = []
        self._account_id = ""

        # caches for when a value is not defined
        self._cached_tick = {}

    def connect(self):
        super().connect()

        try:
            self.lock()

            identity = self.service.identity(self._name)
            self._subscriptions = []  # reset previous list

            if identity:
                self._host = identity.get('host')
                self._account_type = "LIVE" if self._host == "api.ig.com" else "demo"
                self._account_id = identity.get('account-id')

                self._connector = IGConnector(self.service,
                                              identity.get('username'),
                                              identity.get('password'),
                                              identity.get('account-id'),
                                              identity.get('api-key'),
                                              identity.get('host'))

                self._connector.connect()

                # from CST and XST
                password = "******" % (self._connector.cst,
                                              self._connector.xst)
                # logger.debug(self._connector.cst, self._connector.xst, self._connector.lightstreamer_endpoint, identity.get('account-id'), self._connector.client_id)

                if self._lightstreamer:
                    # destroy previous connection
                    self._lightstreamer.destroy()

                self._lightstreamer = LSClient(
                    self._connector.
                    lightstreamer_endpoint,  # "https://push.lightstreamer.com",
                    adapter_set="DEFAULT",
                    user=self._connector.client_id,
                    password=password)

                self._lightstreamer.connect()

                # subscribe for account and trades to have a reactive feedback and don't saturate the REST API
                self.subscribe_account(identity.get('account-id'))
                self.subscribe_trades(identity.get('account-id'))

                #
                # default watched instruments
                #

                all_instruments = []

                if '*' in self.configured_symbols():
                    self._available_instruments = set(all_instruments)
                    instruments = all_instruments
                else:
                    instruments = self.configured_symbols()

                # susbcribe for symbols
                for symbol in instruments:
                    # to know when market close but could be an hourly REST API call, but it consume one subscriber...
                    # @todo so maybe prefers REST call hourly ? but need bid/ofr properly defined at signals on trader.market and strategy.instrument !
                    self.subscribe_market(symbol)

                    # tick data
                    self.subscribe_tick(symbol)

                    # ohlc data (now generated)
                    # for tf in IGWatcher.STORED_TIMEFRAMES:
                    #     self.subscribe_ohlc(symbol, tf)

                    self.insert_watched_instrument(symbol, [0])

            self.service.notify(Signal.SIGNAL_WATCHER_CONNECTED, self.name,
                                time.time())

        except Exception as e:
            logger.debug(repr(e))
            error_logger.error(traceback.format_exc())

            self._connector = None
            self._lightstreamer = None
        finally:
            self.unlock()

    @property
    def connector(self):
        return self._connector

    @property
    def connected(self):
        return self._connector is not None and self._connector.connected

    def subscribe_account(self, account_id):
        fields = [
            "PNL", "AVAILABLE_TO_DEAL", "MARGIN", "FUNDS", "AVAILABLE_CASH"
        ]

        subscription = Subscription(mode="MERGE",
                                    items=["ACCOUNT:" + account_id],
                                    fields=fields,
                                    adapter="")

        self.subscribe(subscription)
        subscription.addlistener(self, IGWatcher.on_account_update)

    def subscribe_trades(self, account_id):
        fields = ["CONFIRMS", "WOU", "OPU"]

        subscription = Subscription(mode="DISTINCT",
                                    items=["TRADE:" + account_id],
                                    fields=fields,
                                    adapter="")

        self.subscribe(subscription)
        subscription.addlistener(self, IGWatcher.on_trade_update)

    def subscribe_tick(self, instrument):
        """
        Subscribe to an instrument tick updates.
        """
        fields = ["BID", "OFR", "LTP", "LTV", "TTV", "UTM"]

        subscription = Subscription(mode="DISTINCT",
                                    items=["CHART:" + instrument + ":TICK"],
                                    fields=fields,
                                    adapter="")

        self.subscribe(subscription)
        subscription.addlistener(self, IGWatcher.on_tick_update)

    # def subscribe_ohlc(self, instrument, timeframe):
    #     """
    #     Subscribe to an instrument. Timeframe must be greater than 0.
    #     """
    #     fields = [
    #         "BID_OPEN", "OFR_OPEN",
    #         "BID_CLOSE", "OFR_CLOSE",
    #         "BID_HIGH", "OFR_HIGH",
    #         "BID_LOW", "OFR_LOW",
    #         "LTP", "LTV", "TTV", "UTM",
    #         "CONS_END"
    #     ]

    #     if timeframe == Instrument.TF_SEC:
    #         tf = "SECOND"
    #     elif timeframe == Instrument.TF_MIN:
    #         tf = "1MINUTE"
    #     elif timeframe == Instrument.TF_5MIN:
    #         tf = "5MINUTE"
    #     elif timeframe == Instrument.TF_HOUR:
    #         tf = "HOUR"
    #     else:
    #         return

    #     subscription = Subscription(
    #         mode="MERGE",
    #         items=["CHART:"+instrument+":"+tf],
    #         fields=fields,
    #         adapter="")

    #     self.subscribe(subscription)
    #     subscription.addlistener(self, IGWatcher.on_ohlc_update)

    def subscribe_market(self, instrument):
        """
        Subscribe to an instrument.
        """
        fields = ["MARKET_STATE", "UPDATE_TIME", "BID", "OFFER"]

        subscription = Subscription(mode="MERGE",
                                    items=["MARKET:" + instrument],
                                    fields=fields,
                                    adapter="")

        self.subscribe(subscription)
        subscription.addlistener(self, IGWatcher.on_market_update)

    def subscribe(self, subscription):
        """
        Registering the Subscription
        """
        sub_key = self._lightstreamer.subscribe(subscription)
        self._subscriptions.append(sub_key)

        return sub_key

    def unsubscribe(self, sub_key):
        if sub_key in self._subscriptions:
            self._lightstreamer.unsubscribe(sub_key)
            del self._subscriptions[sub_key]

    def disconnect(self):
        super().disconnect()

        try:
            self.lock()

            if self._lightstreamer:
                # if self._lightstreamer.connected:
                #   for sub_key in self._subscriptions:
                #       self._lightstreamer.unsubscribe(sub_key)

                self._subscriptions = []
                self._lightstreamer.disconnect()
                self._lightstreamer._join()
                self._lightstreamer = None

            if self._connector:
                self._connector.disconnect()
                self._connector = None

        except Exception as e:
            logger.error(repr(e))
            error_logger.error(traceback.format_exc())
        finally:
            self.unlock()

    def pre_update(self):
        super().pre_update()

        if self._connector is None or not self._connector.connected or self._lightstreamer is None or not self._lightstreamer.connected:
            self._connector = None
            self.connect()

            if not self.connected:
                # retry in 2 second
                time.sleep(2.0)

            return

    def update(self):
        if not super().update():
            return False

        if not self.connected:
            return False

        #
        # ohlc close/open
        #

        self.lock()
        self.update_from_tick()
        self.unlock()

        #
        # market info update (each 4h)
        #

        if time.time(
        ) - self._last_market_update >= IGWatcher.UPDATE_MARKET_INFO_DELAY:  # only once per 4h
            self.update_markets_info()
            self._last_market_update = time.time()

        return True

    def post_update(self):
        super().post_update()
        time.sleep(0.0005)

    def post_run(self):
        super().post_run()

    #
    # WS data
    #

    @staticmethod
    def on_account_update(self, item_update):
        name = item_update.get('name', '').split(':')

        try:
            if len(name) == 2 and name[0] == 'ACCOUNT' and name[
                    1] == self._account_id:
                # live account updates
                values = item_update['values']

                account_data = (float(values['FUNDS']),
                                float(values['AVAILABLE_TO_DEAL']),
                                float(values['PNL']), None, None)
                self.service.notify(Signal.SIGNAL_ACCOUNT_DATA, self.name,
                                    account_data)
        except Exception as e:
            logger.debug(repr(e))
            error_logger.error(traceback.format_exc())

    @staticmethod
    def on_market_update(self, item_update):
        name = item_update.get('name', '').split(':')

        try:
            if len(name) == 2 and name[0] == 'MARKET':
                # market data instrument by epic
                values = item_update['values']
                epic = name[1]

                ready = values['MARKET_STATE'] == 'TRADEABLE'

                # date of the event 20:36:01 without Z
                if ready:
                    update_time = datetime.strptime(values['UPDATE_TIME'],
                                                    '%H:%M:%S').timestamp()
                    market_data = (name[1], True, update_time,
                                   float(values["BID"]),
                                   float(values["OFFER"]), None, None, None,
                                   None, None)
                else:
                    update_time = 0
                    market_data = (name[1], False, 0, 0.0, 0.0, None, None,
                                   None, None, None)

                self.service.notify(Signal.SIGNAL_MARKET_DATA, self.name,
                                    market_data)
        except Exception as e:
            logger.debug(repr(e))
            error_logger.error(traceback.format_exc())

    @staticmethod
    def on_tick_update(self, item_update):
        name = item_update.get('name', '').split(':')

        try:
            if len(name) == 3 and name[0] == 'CHART' and name[2] == 'TICK':
                values = item_update['values']
                market_id = name[1]

                bid = None
                ofr = None
                utm = None
                ltv = None

                if values['UTM']:
                    utm = values['UTM']
                elif market_id in self._cached_tick:
                    utm = self._cached_tick[market_id][0]

                if values['BID']:
                    bid = values['BID']
                elif market_id in self._cached_tick:
                    bid = self._cached_tick[market_id][1]

                if values['OFR']:
                    ofr = values['OFR']
                elif market_id in self._cached_tick:
                    ofr = self._cached_tick[market_id][2]

                if values['LTV']:
                    ltv = values['LTV']
                elif market_id in self._cached_tick:
                    ltv = self._cached_tick[market_id][3]

                if utm is None or bid is None or ofr is None:
                    # need all informations, wait the next one
                    return

                # cache for when a value is not defined
                self._cached_tick[market_id] = (utm, bid, ofr, ltv)

                tick = (float(utm) * 0.001, float(bid), float(ofr),
                        float(ltv or "0"))

                # keep last complete tick values for ohlc generation
                self._last_tick[market_id] = tick

                self.service.notify(Signal.SIGNAL_TICK_DATA, self.name,
                                    (market_id, tick))

                for tf in Watcher.STORED_TIMEFRAMES:
                    # generate candle per each tf
                    self.lock()
                    candle = self.update_ohlc(market_id, tf, tick[0], tick[1],
                                              tick[2], tick[3])
                    self.unlock()

                    if candle is not None:
                        self.service.notify(Signal.SIGNAL_CANDLE_DATA,
                                            self.name, (market_id, candle))

                # disabled for now
                if not self._read_only:
                    Database.inst().store_market_trade(
                        (self.name, market_id, int(utm), bid, ofr, ltv or 0))

        except Exception as e:
            logger.debug(repr(e))
            error_logger.error(traceback.format_exc())

    # @staticmethod
    # def on_ohlc_update(self, item_update):
    #     name = item_update.get('name', '').split(':')

    #     try:
    #         if len(name) == 3 and name[0] == 'CHART':
    #             values = item_update['values']
    #             if values['CONS_END'] == '0':
    #                 # get only consolidated candles
    #                 # @warning It is rarely defined, so many close could be missing, prefers using tick to rebuild ohlc locally
    #                 return

    #             # timeframe
    #             if name[2] == 'SECOND':
    #                 tf = Instrument.TF_SEC
    #             elif name[2] == '1MINUTE':
    #                 tf = Instrument.TF_MIN
    #             elif name[2] == '5MINUTE':
    #                 tf = Instrument.TF_5MIN
    #             elif name[2] == 'HOUR':
    #                 tf = Instrument.TF_HOUR

    #             # one of the value could be missing, use the previous from the cache if we have it
    #             if (values['UTM'] is None or values['LTV'] is None or
    #                 values['OFR_OPEN'] is None or values['OFR_HIGH'] is None or values['OFR_LOW'] is None or values['OFR_CLOSE'] is None or
    #                     values['BID_OPEN'] is None or values['BID_HIGH'] is None or values['BID_LOW'] is None or values['BID_CLOSE'] is None):

    #                 if name[1] not in self._cached_ohlc or tf not in self._cached_ohlc[name[1]]:
    #                     logger.warning("no value and cache miss for %s ohlc in %s (%s)" % (name[1], tf, values))

    #                 if values['UTM'] is None:
    #                     utm = self._cached_ohlc[name[1]][tf][0]
    #                 if values['LTV'] is None:
    #                     ltv = self._cached_ohlc[name[1]][tf][9]

    #             utm = values['UTM']
    #             ltv = values['LTV']

    #             candle = Candle(float(utm) * 0.001, tf)

    #             # if incomplete candle replace ofr by bid or bid by ofr @todo but must be value from previous candle
    #             # but and if we don't have to previous... ok for 1 min but for 1h ? ...
    #             bid_open = values['BID_OPEN'] or values['OFR_OPEN']
    #             bid_high = values['BID_HIGH'] or values['OFR_HIGH']
    #             bid_low = values['BID_LOW'] or values['OFR_LOW']
    #             bid_close = values['BID_CLOSE'] or values['OFR_CLOSE']

    #             ofr_open = values['OFR_OPEN'] or values['BID_OPEN']
    #             ofr_high = values['OFR_HIGH'] or values['BID_HIGH']
    #             ofr_low = values['OFR_LOW'] or values['BID_LOW']
    #             ofr_close = values['OFR_CLOSE'] or values['BID_CLOSE']

    #             candle.set_bid_ohlc(float(bid_open), float(bid_high), float(bid_low), float(bid_close))
    #             candle.set_ofr_ohlc(float(ofr_open), float(ofr_high), float(ofr_low), float(ofr_close))
    #             candle.set_volume(float(values['LTV']) if values['LTV'] else 0.0)
    #             candle.set_consolidated(values['CONS_END'] == '1')

    #             self.service.notify(Signal.SIGNAL_CANDLE_DATA, self.name, (name[1], candle))

    #             if values['CONS_END'] == '1' and not self._read_only:
    #                 # write only consolidated candles. values are string its perfect if not last traded volume then 0
    #                 Database.inst().store_market_ohlc((
    #                     self.name, name[1], int(utm), tf,
    #                     bid_open, bid_high, bid_low, bid_close,
    #                     ofr_open, ofr_high, ofr_low, ofr_close,
    #                     values['LTV'] or "0"))

    #             # cache for when a value is not defined
    #             self._cached_ohlc[name[1]][tf] = (utm, bid_open, bid_high, bid_low, bid_close, ofr_open, ofr_high, ofr_low, ofr_close, ltv)

    #     except Exception as e:
    #         logger.debug(repr(e))
    #         error_logger.error(traceback.format_exc())

    @staticmethod
    def on_trade_update(self, item_update):
        name = item_update.get('name', '').split(':')

        try:
            if len(name) == 2 and name[0] == 'TRADE' and name[
                    1] == self._account_id:
                # live trade updates
                values = item_update['values']

                #
                # order confirms (accepted/rejected)
                #

                if values.get('CONFIRMS'):
                    # not use them because we only want CRUD operations => OPU only so
                    data = json.loads(values.get('CONFIRMS'))
                    exec_logger.info("ig.com CONFIRMS %s" % str(data))

                    epic = data.get('epic')
                    level = float(data['level']) if data.get(
                        'level') is not None else None
                    quantity = float(
                        data['size']) if data.get('size') is not None else None

                    if data['dealStatus'] == 'REJECTED':
                        ref_order_id = data['dealReference']

                        # if data['reason'] == 'INSUFFICIENT_BALANCE':
                        #   reason = 'insufficient balance'

                        self.service.notify(Signal.SIGNAL_ORDER_REJECTED,
                                            self.name, (epic, ref_order_id))

                    elif data['dealStatus'] == 'ACCEPTED':
                        # deal confirmed and accepted
                        order_id = data['dealId']
                        ref_order_id = data['dealReference']

                        exec_logger.info("ig 538 'CONFIRMS' %s" % str(data))

                        # date 2018-09-13T20:36:01.096 without Z
                        event_time = datetime.strptime(
                            data['date'], '%Y-%m-%dT%H:%M:%S.%f').timestamp()

                        if data['direction'] == 'BUY':
                            direction = Order.LONG
                        elif data['direction'] == 'SELL':
                            direction = Order.SHORT
                        else:
                            direction = Order.LONG

                        quantity = float(data.get('size')) if data.get(
                            'size') is not None else 0.0

                        # don't send event because create_order return True in that case

                        if quantity and level:
                            # signal of updated order
                            order_data = {
                                'id': order_id,
                                # @todo important we want TRADED and UPDATED distinct
                            }

                            # @todo to be completed before signal, but not really necessary we can work with position update
                            # self.service.notify(Signal.SIGNAL_ORDER_TRADED, self.name, (epic, order_data, ref_order_id))

                #
                # active waiting order (open/updated/deleted)
                #

                if values.get('WOU'):
                    data = json.loads(values.get('WOU'))
                    exec_logger.info("ig.com WOU %s" % str(data))

                    order_id = data['dealId']
                    ref_order_id = data['dealReference']

                    epic = data['epic']

                    # level = float(data['level']) if data.get('level') is not None else None
                    # stop_level = float(data['stopLevel']) if data.get('stopLevel') is not None else None
                    # limit_level = float(data['limitLevel']) if data.get('limitLevel') is not None else None
                    # stop_distance = float(data['stopDistance']) if data.get('stopDistance') is not None else None
                    # limit_distance = float(data['limitDistance']) if data.get('limitDistance') is not None else None
                    # profit_loss = float(data['profit']) if data.get('profit') is not None else 0.0
                    # epic, level, guaranteedStop, currency, timeInForce (GOOD_TILL_CANCELLED, GOOD_TILL_DATE)
                    # goodTillDate, size, timestamp, stopDistance, limitDistance

                    # if data['orderType'] == "LIMIT":
                    #   pass
                    # elif data['orderType'] == "STOP":
                    #   pass

                    # status OPEN, DELETED, FULLY_CLOSED
                    if data['status'] == "OPEN":
                        order_data = {
                            'id': order_id,
                            # @todo important
                        }

                        self.service.notify(Signal.SIGNAL_ORDER_OPENED,
                                            self.name,
                                            (epic, order_data, ref_order_id))

                    elif data['status'] == "UPDATED":
                        # signal of updated order
                        order_data = {
                            'id': order_id,
                            # @todo important we want TRADED and UPDATED distinct
                        }

                        self.service.notify(Signal.SIGNAL_ORDER_UPDATED,
                                            self.name,
                                            (epic, order_data, ref_order_id))

                    elif data['status'] == "DELETED":
                        # signal of deleted order
                        self.service.notify(Signal.SIGNAL_ORDER_DELETED,
                                            self.name,
                                            (epic, order_id, ref_order_id))

                    elif data['status'] == "FULLY_CLOSED":
                        # @todo ??
                        pass

                #
                # active position (open/updated/deleted)
                #

                if values.get('OPU'):
                    data = json.loads(values.get('OPU'))
                    exec_logger.info("ig.com OPU %s" % str(data))

                    if data.get('direction', '') == 'BUY':
                        direction = Order.LONG
                    elif data.get('direction', '') == 'SELL':
                        direction = Order.SHORT
                    else:
                        direction = Order.LONG

                    position_id = data['dealId']
                    ref_order_id = data['dealReference']

                    epic = data.get('epic')
                    quantity = float(data.get('size')) if data.get(
                        'size') is not None else 0.0

                    level = float(data['level']) if data.get(
                        'level') is not None else None
                    stop_level = float(data['stopLevel']) if data.get(
                        'stopLevel') is not None else None
                    limit_level = float(data['limitLevel']) if data.get(
                        'limitLevel') is not None else None
                    stop_distance = float(data['stopDistance']) if data.get(
                        'stopDistance') is not None else None
                    limit_distance = float(data['limitDistance']) if data.get(
                        'limitDistance') is not None else None
                    profit_loss = float(data['profit']) if data.get(
                        'profit') is not None else 0.0

                    # "dealStatus": "ACCEPTED",
                    # "channel": "WTP", "expiry": "-", "currency": "EUR", "guaranteedStop": false,
                    # @todo "orderType": "LIMIT", "timeInForce": "GOOD_TILL_CANCELLED", "goodTillDate": null

                    # date of the event 2018-09-13T20:36:01.096 without Z
                    event_time = datetime.strptime(
                        data['timestamp'], '%Y-%m-%dT%H:%M:%S.%f').timestamp()

                    # status OPEN, UPDATED, DELETED
                    if data['status'] == "OPEN":
                        # signal of opened position
                        position_data = {
                            'id': position_id,
                            'symbol': epic,
                            'direction': direction,
                            'timestamp': event_time,
                            'quantity': quantity,
                            'exec-price': level,
                            'stop-loss': stop_distance,
                            'take-profit': limit_level,
                            'profit-loss': profit_loss,
                            'cumulative-filled': quantity,
                            'filled': None,  # no have
                            'liquidation-price': None  # no have
                        }

                        self.service.notify(
                            Signal.SIGNAL_POSITION_OPENED, self.name,
                            (epic, position_data, ref_order_id))

                    elif data['status'] == "UPDATED":
                        # signal of updated position
                        position_data = {
                            'id': position_id,
                            'symbol': epic,
                            'direction': direction,
                            'timestamp': event_time,
                            'quantity': quantity,
                            'exec-price': level,
                            'stop-loss': stop_distance,
                            'take-profit': limit_level,
                            # 'profit-currency': '', 'profitCurrency'
                            'profit-loss': profit_loss,
                            # @todo trailingStep, trailingStopDistance, guaranteedStop
                            'cumulative-filled': quantity,
                            'filled': None,  # no have
                            'liquidation-price': None  # no have
                        }

                        self.service.notify(
                            Signal.SIGNAL_POSITION_UPDATED, self.name,
                            (epic, position_data, ref_order_id))

                    elif data['status'] == "DELETED":
                        # signal of updated position
                        position_data = {
                            'id': position_id,
                            'symbol': epic,
                            'direction': direction,
                            'timestamp': event_time,
                            'quantity': quantity,
                            'exec-price': level,
                            'stop-loss': stop_distance,
                            'take-profit': limit_level,
                            # 'profit-currency': '', 'profitCurrency'
                            'profit-loss': profit_loss,
                            # @todo trailingStep, trailingStopDistance, guaranteedStop
                            'cumulative-filled': quantity,
                            'filled': None,  # no have
                            'liquidation-price': None  # no have
                        }

                        self.service.notify(
                            Signal.SIGNAL_POSITION_DELETED, self.name,
                            (epic, position_data, ref_order_id))
                    else:
                        exec_logger.info("ig l695 'OPU' %s" % str(data))

        except Exception as e:
            logger.debug(repr(e))
            exec_logger.error(traceback.format_exc())

    #
    # REST data
    #

    def fetch_market(self, epic):
        """
        Fetch and cache it. It rarely changes, except for base exchange rate, so assume it once for all.
        """
        market_info = self._connector.market(epic)

        instrument = market_info['instrument']
        snapshot = market_info['snapshot']
        dealing_rules = market_info['dealingRules']

        market = Market(epic, instrument['marketId'])

        # cannot interpret this value because IG want it as it is
        market.expiry = instrument['expiry']

        # not perfect but IG does not provides this information
        if instrument["marketId"].endswith(
                instrument["currencies"][0]["name"]):
            base_symbol = instrument[
                "marketId"][:-len(instrument["currencies"][0]["name"])]
        else:
            base_symbol = instrument["marketId"]

        market.base_exchange_rate = instrument['currencies'][0][
            'baseExchangeRate']  # "exchangeRate": 0.77

        market.one_pip_means = float(instrument['onePipMeans'].split(' ')[0])
        market.value_per_pip = float(instrument['valueOfOnePip'])
        market.contract_size = float(instrument['contractSize'])
        market.lot_size = float(instrument['lotSize'])

        # @todo how to determine base precision ?
        market.set_base(base_symbol, base_symbol)
        market.set_quote(instrument["currencies"][0]["name"],
                         instrument["currencies"][0]['symbol'],
                         decimal_place(market.one_pip_means))  # "USD", "$"

        if snapshot:
            market.is_open = snapshot["marketStatus"] == "TRADEABLE"
            market.bid = snapshot['bid']
            market.ofr = snapshot['offer']

        # "marginFactorUnit": "PERCENTAGE" not avalaible if market is down
        if instrument.get('marginFactor') and market.is_open:
            market.margin_factor = float(instrument['marginFactor'])
            margin_factor = instrument['marginFactor']
        elif instrument.get('margin') and market.is_open:
            market.margin_factor = 0.1 / float(instrument['margin'])
            margin_factor = str(market.margin_factor)
        else:
            # we don't want this when market is down because it could overwrite the previous stored value
            margin_factor = None

        if instrument['unit'] == 'AMOUNT':
            market.unit_type = Market.UNIT_AMOUNT
        elif instrument['unit'] == 'CONTRACTS':
            market.unit_type = Market.UNIT_CONTRACTS
        elif instrument['unit'] == 'SHARES':
            market.unit_type = Market.UNIT_SHARES

        # BINARY OPT_* BUNGEE_*
        if instrument['type'] == 'CURRENCIES':
            market.market_type = Market.TYPE_CURRENCY
        elif instrument['type'] == 'INDICES':
            market.market_type = Market.TYPE_INDICE
        elif instrument['type'] == 'COMMODITIES':
            market.market_type = Market.TYPE_COMMODITY
        elif instrument['type'] == 'SHARES':
            market.market_type = Market.TYPE_STOCK
        elif instrument['type'] == 'RATES':
            market.market_type = Market.TYPE_RATE
        elif instrument['type'] == 'SECTORS':
            market.market_type = Market.TYPE_SECTOR

        market.trade = Market.TRADE_MARGIN
        market.contract_type = Market.CONTRACT_CFD

        # take minDealSize as tick size
        market.set_size_limits(dealing_rules["minDealSize"]["value"], 0.0,
                               dealing_rules["minDealSize"]["value"])
        # @todo there is some limits in contract size
        market.set_notional_limits(0.0, 0.0, 0.0)
        # @todo maybe decimal_place of onePipMeans for tick_size
        market.set_price_limits(0.0, 0.0, 0.0)

        # commission for stocks
        commission = "0.0"
        # @todo

        # store the last market info to be used for backtesting
        if not self._read_only:
            Database.inst().store_market_info((
                self.name,
                epic,
                market.symbol,
                market.market_type,
                market.unit_type,
                market.contract_type,  # type
                market.trade,
                market.orders,  # type
                market.base,
                market.base_display,
                market.base_precision,  # base
                market.quote,
                market.quote_display,
                market.quote_precision,  # quote
                market.expiry,
                int(market.last_update_time * 1000.0),  # expiry, timestamp
                instrument['lotSize'],
                instrument['contractSize'],
                str(market.base_exchange_rate),
                instrument['valueOfOnePip'],
                instrument['onePipMeans'].split(' ')[0],
                margin_factor,
                dealing_rules["minDealSize"]["value"],
                "0.0",
                dealing_rules["minDealSize"]["value"],  # size limits
                "0.0",
                "0.0",
                "0.0",  # notional limits
                "0.0",
                "0.0",
                "0.0",  # price limits
                "0.0",
                "0.0",
                commission,
                commission)  # fees
                                              )

        return market

    def update_markets_info(self):
        """
        Update market info (very important because IG frequently changes lot or contract size).
        """
        for market_id in self._watched_instruments:
            market = self.fetch_market(market_id)

            if market.is_open:
                market_data = (market_id, market.is_open,
                               market.last_update_time, market.bid, market.ofr,
                               market.base_exchange_rate, market.contract_size,
                               market.value_per_pip, market.vol24h_base,
                               market.vol24h_quote)
            else:
                market_data = (market_id, market.is_open,
                               market.last_update_time, 0.0, 0.0, None, None,
                               None, None, None)

            self.service.notify(Signal.SIGNAL_MARKET_DATA, self.name,
                                market_data)
コード例 #3
0
class IGFetcher(Fetcher):
    """
    IG watcher data fetcher.
    @noto Limitation on 10000 candles per week.
    """

    def __init__(self, service):
        super().__init__("ig.com", service)

        self._host = "ig.com"
        self._connector = None
        self._account_id = ""

    def connect(self):
        super().connect()

        try:
            identity = self.service.identity(self._name)

            if identity:
                self._host = identity.get('host')
                self._account_type = "LIVE" if self._host == "api.ig.com" else "demo"
                self._account_id = identity.get('account-id')

                self._connector = IGConnector(
                    self.service,
                    identity.get('username'),
                    identity.get('password'),
                    identity.get('account-id'),
                    identity.get('api-key'),
                    identity.get('host'))

                self._connector.connect()

                self._available_instruments = set()

        except Exception as e:
            logger.error(repr(e))
            error_logger.error(traceback.format_exc())

            self._connector = None

    @property
    def connector(self):
        return self._connector

    @property
    def connected(self):
        return self._connector is not None and self._connector.connected

    def disconnect(self):
        super().disconnect()

        try:
            if self._connector:
                self._connector.disconnect()
                self._connector = None

        except Exception as e:
            logger.error(repr(e))
            error_logger.error(traceback.format_exc())

    def has_instrument(self, instrument, fetch_option=""):
        return True  # @todo check...

    def fetch_trades(self, market_id, from_date=None, to_date=None, n_last=None, fetch_option=""):
        pass

    def fetch_candles(self, market_id, timeframe, from_date=None, to_date=None, n_last=None, fetch_option=""):
        try:
            if n_last:
                data = self._connector.history_last_n(market_id, timeframe, n_last)
            else:
                data = self._connector.history_range(market_id, timeframe, from_date, to_date)
        except Exception as e:
            logger.error(repr(e))
            error_logger.error(traceback.format_exc())

            data = {}

        prices = data.get('prices', [])

        for price in prices:
            timestamp = datetime.strptime(price['snapshotTime'], '%Y:%m:%d-%H:%M:%S').timestamp()

            if price.get('highPrice')['bid'] is None and price.get('highPrice')['ask'] is None:
                # ignore empty candles
                continue

            # yield (timestamp, high bid, low, open, close, high ofr, low, open, close, volume)
            yield([int(timestamp * 1000),
                str(price.get('highPrice')['bid'] or price.get('highPrice')['ask']),
                str(price.get('lowPrice')['bid'] or price.get('lowPrice')['ask']),
                str(price.get('openPrice')['bid'] or price.get('openPrice')['ask']),
                str(price.get('closePrice')['bid'] or price.get('closePrice')['ask']),
                str(price.get('highPrice')['ask'] or price.get('highPrice')['bid']),
                str(price.get('lowPrice')['ask'] or price.get('lowPrice')['bid']),
                str(price.get('openPrice')['ask'] or price.get('openPrice')['bid']),
                str(price.get('closePrice')['ask'] or price.get('closePrice')['bid']),
                price.get('lastTradedVolume', '0')])
コード例 #4
0
ファイル: fetcher.py プロジェクト: venkiiee/siis
class IGFetcher(Fetcher):
    """
    IG watcher data fetcher.
    @note Initial limitation to 10000 candles per week.
    @warning UTC timestamp are erroneous in D, W and M because there is an issue with DST changes.
    Then have to fix that cases.
    """

    def __init__(self, service):
        super().__init__("ig.com", service)

        self._host = "ig.com"
        self._connector = None
        self._account_id = ""
        self._tzname = None

    def connect(self):
        super().connect()

        try:
            identity = self.service.identity(self._name)

            if identity:
                self._host = identity.get('host')
                self._account_type = "LIVE" if self._host == "api.ig.com" else "demo"
                self._account_id = identity.get('account-id')

                self._connector = IGConnector(
                    self.service,
                    identity.get('username'),
                    identity.get('password'),
                    identity.get('account-id'),
                    identity.get('api-key'),
                    identity.get('host'))

                self._tzname = identity.get('tzname')
                self._connector.connect()

                self._available_instruments = set()

        except Exception as e:
            logger.error(repr(e))
            error_logger.error(traceback.format_exc())

            self._connector = None

    @property
    def connector(self):
        return self._connector

    @property
    def connected(self):
        return self._connector is not None and self._connector.connected

    def disconnect(self):
        super().disconnect()

        try:
            if self._connector:
                self._connector.disconnect()
                self._connector = None

        except Exception as e:
            logger.error(repr(e))
            error_logger.error(traceback.format_exc())

    def has_instrument(self, instrument, fetch_option=""):
        return True  # @todo check...

    def fetch_trades(self, market_id, from_date=None, to_date=None, n_last=None, fetch_option=""):
        pass

    def fetch_candles(self, market_id, timeframe, from_date=None, to_date=None, n_last=None, fetch_option=""):
        try:
            if n_last:
                data = self._connector.history_last_n(market_id, timeframe, n_last)
            else:
                data = self._connector.history_range(market_id, timeframe, from_date, to_date)
        except Exception as e:
            logger.error(repr(e))
            error_logger.error(traceback.format_exc())

            data = {}

        prices = data.get('prices', [])

        # get local timezone, assume its the same of the account, or overrided by account detail
        tzname = self._tzname or time.tzname[0]
        pst = pytz.timezone(tzname)

        for price in prices:
            dt = datetime.strptime(price['snapshotTimeUTC'], '%Y-%m-%dT%H:%M:%S').replace(tzinfo=UTC())
            # ldt = datetime.strptime(price['snapshotTime'], '%Y/%m/%d %H:%M:%S')

            # timezone + DST aware conversion
            # print("<", dt, ldt)
            # dt = dt + pst.localize(ldt).dst() + pst.localize(ldt).utcoffset()

            # fix for D,W,M snapshotTimeUTC, probaby because of the DST (then might be +1 or -1 hour)
            if timeframe in (Instrument.TF_1D, Instrument.TF_1W, Instrument.TF_1M):
                if dt.hour == 23:
                    # is 23:00 on the previous day, add 1h
                    dt = dt + timedelta(hours=1)
                elif dt.hour == 1:
                    # is 01:00 on the same day, sub 1h
                    dt = dt - timedelta(hours=1)

            elif timeframe == Instrument.TF_4H:
                if dt.hour in (3, 7, 11, 15, 19, 23):
                    dt = dt + timedelta(hours=1)
                elif dt.hour in (1, 5, 9, 13, 17, 21):
                     dt = dt - timedelta(hours=1)

            # print(">", dt, ldt)
            timestamp = dt.timestamp()

            if price.get('highPrice')['bid'] is None and price.get('highPrice')['ask'] is None:
                # ignore empty candles
                continue

            # yield (timestamp, high bid, low, open, close, high ofr, low, open, close, volume)
            yield([int(timestamp * 1000),
                str(price.get('openPrice')['bid'] or price.get('openPrice')['ask']),
                str(price.get('highPrice')['bid'] or price.get('highPrice')['ask']),
                str(price.get('lowPrice')['bid'] or price.get('lowPrice')['ask']),
                str(price.get('closePrice')['bid'] or price.get('closePrice')['ask']),
                str(price.get('openPrice')['ask'] or price.get('openPrice')['bid']),
                str(price.get('highPrice')['ask'] or price.get('highPrice')['bid']),
                str(price.get('lowPrice')['ask'] or price.get('lowPrice')['bid']),
                str(price.get('closePrice')['ask'] or price.get('closePrice')['bid']),
                price.get('lastTradedVolume', '0')])