def update_ohlc(self, market_id, tf, ts, bid, ofr, volume): base_time = Instrument.basetime(ts, time.time()) ended_ohlc = None ohlc = None # last ohlc per market id last_ohlc_by_timeframe = self._last_ohlc.get(market_id) if last_ohlc_by_timeframe is None: # not found for this market insert it self._last_ohlc[market_id] = {tf: None} last_ohlc_by_timeframe = self._last_ohlc[market_id] if tf not in last_ohlc_by_timeframe: last_ohlc_by_timeframe[tf] = None else: ohlc = last_ohlc_by_timeframe[tf] if ohlc and (ohlc.timestamp + tf <= base_time): # later tick data (laggy ?) if ts < base_time: # but after that next in laggy data will be ignored... # its not perfect in laggy cases for storage, but in live we can't deals we later data self.__update_ohlc(ohlc, bid, ofr, volume) # need to close the ohlc and to open a new one ohlc.set_consolidated(True) ended_ohlc = ohlc last_ohlc_by_timeframe[tf] = None ohlc = None if ohlc is None: # open a new one ohlc = Candle(base_time, tf) ohlc.set_consolidated(False) if bid: ohlc.set_bid_ohlc(bid, bid, bid, bid) if ofr: ohlc.set_ofr_ohlc(ofr, ofr, ofr, ofr) last_ohlc_by_timeframe[tf] = ohlc if ts >= ohlc.timestamp: self.__update_ohlc(ohlc, bid, ofr, volume) # stored timeframes only if ended_ohlc and (tf in self.STORED_TIMEFRAMES): Database.inst().store_market_ohlc( (self.name, market_id, int(ended_ohlc.timestamp * 1000), tf, ended_ohlc.bid_open, ended_ohlc.bid_high, ended_ohlc.bid_low, ended_ohlc.bid_close, ended_ohlc.ofr_open, ended_ohlc.ofr_high, ended_ohlc.ofr_low, ended_ohlc.ofr_close, ended_ohlc.volume)) return ohlc
def update_from_tick(self): """ During update processing, close OHLCs if not tick data arrive before. Then notify a signal if a ohlc is generated (and closed). """ now = time.time() for tf in self.GENERATED_TF: # only if current base time is greater than the previous base_time = Instrument.basetime(tf, now) if base_time > self._last_update_times[tf]: self._last_update_times[tf] = base_time for market_id, last_ohlc_by_timeframe in self._last_ohlc.items( ): if last_ohlc_by_timeframe: ohlc = self.close_ohlc(market_id, last_ohlc_by_timeframe, tf, now) if ohlc: self.service.notify(Signal.SIGNAL_CANDLE_DATA, self.name, (market_id, ohlc))
def write(self, msg): Instrument.write(self, '+eoi:0') # Disable EOI Instrument.write(self, '+eos:13') # Set EOS char to CR (ASCII 13) Instrument.write(self, msg) Instrument.write(self, '+eoi:1') # Enable EOI
def process_ohlc(self): # # select market ohlcs # with self._mutex: mks = self._pending_ohlc_select self._pending_ohlc_select = [] if mks: try: cursor = self._db.cursor() for mk in mks: if mk[6]: # last n cursor.execute("""SELECT COUNT(*) FROM ohlc WHERE broker_id = '%s' AND market_id = '%s' AND timeframe = %s""" % (mk[1], mk[2], mk[3])) count = int(cursor.fetchone()[0]) offset = max(0, count - mk[6]) # LIMIT should not be necessary then cursor.execute("""SELECT timestamp, bid_open, bid_high, bid_low, bid_close, ask_open, ask_high, ask_low, ask_close, volume FROM ohlc WHERE broker_id = '%s' AND market_id = '%s' AND timeframe = %s ORDER BY timestamp ASC LIMIT %i OFFSET %i""" % ( mk[1], mk[2], mk[3], mk[6], offset)) elif mk[4] and mk[5]: # from to cursor.execute("""SELECT timestamp, bid_open, bid_high, bid_low, bid_close, ask_open, ask_high, ask_low, ask_close, volume FROM ohlc WHERE broker_id = '%s' AND market_id = '%s' AND timeframe = %s AND timestamp >= %i AND timestamp <= %i ORDER BY timestamp ASC""" % ( mk[1], mk[2], mk[3], mk[4], mk[5])) elif mk[4]: # from to now cursor.execute("""SELECT timestamp, bid_open, bid_high, bid_low, bid_close, ask_open, ask_high, ask_low, ask_close, volume FROM ohlc WHERE broker_id = '%s' AND market_id = '%s' AND timeframe = %s AND timestamp >= %i ORDER BY timestamp ASC""" % ( mk[1], mk[2], mk[3], mk[4])) elif mk[5]: # to now cursor.execute("""SELECT timestamp, bid_open, bid_high, bid_low, bid_close, ask_open, ask_high, ask_low, ask_close, volume FROM ohlc WHERE broker_id = '%s' AND market_id = '%s' AND timeframe = %s AND timestamp <= %i ORDER BY timestamp ASC""" % ( mk[1], mk[2], mk[3], mk[5])) else: # all cursor.execute("""SELECT timestamp, bid_open, bid_high, bid_low, bid_close, ask_open, ask_high, ask_low, ask_close, volume FROM ohlc WHERE broker_id = '%s' AND market_id = '%s' AND timeframe = %s ORDER BY timestamp ASC""" % ( mk[1], mk[2], mk[3])) rows = cursor.fetchall() ohlcs = [] for row in rows: timestamp = float(row[0]) * 0.001 # to float second timestamp ohlc = Candle(timestamp, mk[3]) ohlc.set_bid_ohlc(float(row[1]), float(row[2]), float(row[3]), float(row[4])) ohlc.set_ofr_ohlc(float(row[5]), float(row[6]), float(row[7]), float(row[8])) # if float(row[9]) <= 0: # # prefer to ignore empty volume ohlc because it can broke volume signal and it is a no way but it could be # # a lack of this information like on SPX500 of ig.com. So how to manage that cases... # continue ohlc.set_volume(float(row[9])) if ohlc.timestamp >= Instrument.basetime(mk[3], time.time()): ohlc.set_consolidated(False) # current ohlcs.append(ohlc) # notify mk[0].notify(Signal.SIGNAL_CANDLE_DATA_BULK, mk[1], (mk[2], mk[3], ohlcs)) except self.psycopg2.OperationalError as e: self.try_reconnect(e) # retry the next time with self._mutex: self._pending_ohlc_select = mks + self._pending_ohlc_select except Exception as e: self.on_error(e) # retry the next time with self._mutex: self._pending_ohlc_select = mks + self._pending_ohlc_select # # insert market ohlcs # if time.time() - self._last_ohlc_flush >= 60 or len(self._pending_ohlc_insert) > 500: with self._mutex: mkd = self._pending_ohlc_insert self._pending_ohlc_insert = [] if mkd: try: cursor = self._db.cursor() elts = [] data = set() for mk in mkd: if (mk[0], mk[1], mk[2], mk[3]) not in data: elts.append("('%s', '%s', %i, %i, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')" % (mk[0], mk[1], mk[2], mk[3], mk[4], mk[5], mk[6], mk[7], mk[8], mk[9], mk[10], mk[11], mk[12])) data.add((mk[0], mk[1], mk[2], mk[3])) query = ' '.join(("INSERT INTO ohlc(broker_id, market_id, timestamp, timeframe, bid_open, bid_high, bid_low, bid_close, ask_open, ask_high, ask_low, ask_close, volume) VALUES", ','.join(elts), "ON CONFLICT (broker_id, market_id, timestamp, timeframe) DO UPDATE SET bid_open = EXCLUDED.bid_open, bid_high = EXCLUDED.bid_high, bid_low = EXCLUDED.bid_low, bid_close = EXCLUDED.bid_close, ask_open = EXCLUDED.ask_open, ask_high = EXCLUDED.ask_high, ask_low = EXCLUDED.ask_low, ask_close = EXCLUDED.ask_close, volume = EXCLUDED.volume")) # query = ' '.join(( # "INSERT INTO ohlc(broker_id, market_id, timestamp, timeframe, bid_open, bid_high, bid_low, bid_close, ask_open, ask_high, ask_low, ask_close, volume) VALUES", # ','.join(["('%s', '%s', %i, %i, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')" % (mk[0], mk[1], mk[2], mk[3], mk[4], mk[5], mk[6], mk[7], mk[8], mk[9], mk[10], mk[11], mk[12]) for mk in mkd]), # "ON CONFLICT (broker_id, market_id, timestamp, timeframe) DO UPDATE SET bid_open = EXCLUDED.bid_open, bid_high = EXCLUDED.bid_high, bid_low = EXCLUDED.bid_low, bid_close = EXCLUDED.bid_close, ask_open = EXCLUDED.ask_open, ask_high = EXCLUDED.ask_high, ask_low = EXCLUDED.ask_low, ask_close = EXCLUDED.ask_close, volume = EXCLUDED.volume" # )) cursor.execute(query) self._db.commit() except self.psycopg2.OperationalError as e: self.try_reconnect(e) # retry the next time with self._mutex: self._pending_ohlc_insert = mkd + self._pending_ohlc_insert except Exception as e: self.on_error(e) # retry the next time with self._mutex: self._pending_ohlc_insert = mkd + self._pending_ohlc_insert self._last_ohlc_flush = time.time() # # insert market liquidation # with self._mutex: mkd = self._pending_liquidation_insert self._pending_liquidation_insert = [] if mkd: try: cursor = self._db.cursor() elts = [] for mk in mkd: elts.append("('%s', '%s', %i, %i, '%s', '%s')" % (mk[0], mk[1], mk[2], mk[3], mk[4], mk[5])) # query = ' '.join(("INSERT INTO liquidation(broker_id, market_id, timestamp, direction, price, quantity) VALUES", # ','.join(elts), # "ON CONFLICT (broker_id, market_id, timestamp) DO NOTHING")) query = ' '.join(("INSERT INTO liquidation(broker_id, market_id, timestamp, direction, price, quantity) VALUES", ','.join(elts))) cursor.execute(query) self._db.commit() except psycopg2.OperationalError as e: self.try_reconnect(e) # retry the next time with self._mutex: self._pending_liquidation_insert = mkd + self._pending_liquidation_insert except Exception as e: self.on_error(e) # retry the next time with self._mutex: self._pending_liquidation_insert = mkd + self._pending_liquidation_insert # # clean older ohlcs # if self._autocleanup: if time.time() - self._last_ohlc_clean >= OhlcStorage.CLEANUP_DELAY: try: now = time.time() cursor = self._db.cursor() for timeframe, timestamp in OhlcStorage.CLEANERS: ts = int(now - timestamp) * 1000 # @todo make a count before cursor.execute("DELETE FROM ohlc WHERE timeframe <= %i AND timestamp < %i" % (timeframe, ts)) self._db.commit() except psycopg2.OperationalError as e: self.try_reconnect(e) except Exception as e: self.on_error(e) self._last_ohlc_clean = time.time()
def write(self, msg): Instrument.write(self, '+eoi:0') # Disable EOI Instrument.write(self, '+eos:13') # Set EOS char to CR (ASCII 13) Instrument.write(self, msg) Instrument.write(self, '+eoi:1') # Enable EOI
def fetch_and_generate(self, market_id, timeframe, n_last=1, cascaded=None): """ For initial fetching of the current OHLC. """ if timeframe > 0 and timeframe not in self.GENERATED_TF: logger.error("Timeframe %i is not allowed !" % (timeframe, )) return generators = [] from_tf = timeframe if not market_id in self._last_ohlc: self._last_ohlc[market_id] = {} # compute a from date today = datetime.now().astimezone(UTC()) from_date = today - timedelta(seconds=timeframe * n_last) to_date = today last_ohlcs = {} # cascaded generation of candles if cascaded: for tf in Watcher.GENERATED_TF: if tf > timeframe: # from timeframe greater than initial if tf <= cascaded: # until max cascaded timeframe generators.append(CandleGenerator(from_tf, tf)) from_tf = tf # store for generation last_ohlcs[tf] = [] else: from_tf = tf if timeframe > 0: last_ohlcs[timeframe] = [] n = 0 for data in self.fetch_candles(market_id, timeframe, from_date, to_date, None): # store (int timestamp in ms, str bid, str ofr, str volume) if not self._read_only: Database.inst().store_market_trade( (self.name, market_id, data[0], data[1], data[2], data[3])) candle = Candle(float(data[0]) * 0.001, timeframe) candle.set_bid_ohlc(float(data[1]), float(data[2]), float(data[3]), float(data[4])) candle.set_ofr_ohlc(float(data[5]), float(data[6]), float(data[7]), float(data[8])) candle.set_volume(float(data[9])) if candle.timestamp >= Instrument.basetime(timeframe, time.time()): candle.set_consolidated(False) # current last_ohlcs[timeframe].append(candle) # only the last self._last_ohlc[market_id][timeframe] = candle # generate higher candles for generator in generators: candles = generator.generate_from_candles( last_ohlcs[generator.from_tf], False) if candles: if not self._read_only: for c in candles: self.store_candle(market_id, generator.to_tf, c) last_ohlcs[generator.to_tf].extend(candles) # only the last as current self._last_ohlc[market_id][generator.to_tf] = candles[-1] elif generator.current: self._last_ohlc[market_id][ generator.to_tf] = generator.current # remove consumed candles last_ohlcs[generator.from_tf] = [] n += 1 for k, ohlc in self._last_ohlc[market_id].items(): if ohlc: ohlc.set_consolidated(False)
def update_ohlc(self, market_id, tf, ts, bid, ofr, volume): """ Update the current OHLC or create a new one, and save them. @param market_id str Unique market identifier @param tf float Timeframe (normalized timeframe at second) @param ts float Timestamp of the update or of the tick/trade @param bid float Bid price. @param ofr float Offer/ask price. @param volume float Volume transacted or 0 if unspecified. """ ended_ohlc = None ohlc = None # last ohlc per market id last_ohlc_by_timeframe = self._last_ohlc.get(market_id) if last_ohlc_by_timeframe is None: # not found for this market insert it self._last_ohlc[market_id] = {tf: None} last_ohlc_by_timeframe = self._last_ohlc[market_id] if tf not in last_ohlc_by_timeframe: last_ohlc_by_timeframe[tf] = None else: ohlc = last_ohlc_by_timeframe[tf] if ohlc and ts >= ohlc.timestamp + tf: # need to close the current ohlc ohlc.set_consolidated(True) ended_ohlc = ohlc last_ohlc_by_timeframe[tf] = None ohlc = None if ohlc is None: # open a new one if necessary base_time = Instrument.basetime(tf, ts) ohlc = Candle(base_time, tf) ohlc.set_consolidated(False) if bid: ohlc.set_bid(bid) if ofr: ohlc.set_ofr(ofr) last_ohlc_by_timeframe[tf] = ohlc if ts >= ohlc.timestamp: # update the current OHLC if volume: ohlc._volume += volume if bid: if not ohlc._bid_open: ohlc.set_bid(bid) # update bid prices ohlc._bid_high = max(ohlc._bid_high, bid) ohlc._bid_low = min(ohlc._bid_low, bid) # potential close ohlc._bid_close = bid if ofr: if not ohlc.ofr_open: ohlc.set_ofr(ofr) # update ofr prices ohlc._ofr_high = max(ohlc._ofr_high, ofr) ohlc._ofr_low = min(ohlc._ofr_low, ofr) # potential close ohlc._ofr_close = ofr # stored timeframes only if ended_ohlc and (tf in self.STORED_TIMEFRAMES): Database.inst().store_market_ohlc( (self.name, market_id, int(ended_ohlc.timestamp * 1000), tf, ended_ohlc.bid_open, ended_ohlc.bid_high, ended_ohlc.bid_low, ended_ohlc.bid_close, ended_ohlc.ofr_open, ended_ohlc.ofr_high, ended_ohlc.ofr_low, ended_ohlc.ofr_close, ended_ohlc.volume)) return ohlc
def basetime(self): """ Related candle base time of the timestamp of the signal. """ return Instrument.basetime(self.timeframe, self.ts)
def compute(self, timestamp, timestamps, high, low, close): size = len(close) basis = ta_SMA(close, self._length) atrs = ta_ATR(high, low, close, timeperiod=self._length) dev = atrs * self._coeff upper = basis + dev lower = basis - dev with np.errstate(divide='ignore', invalid='ignore'): # replace by 0 when divisor is 0 bbr = (close - lower) / (upper - lower) bbr[bbr == np.inf] = 0.0 bbe = ta_EMA(bbr, self._length_MA) if len(self._tup) != size: self._tup = np.zeros(size) self._tdn = np.zeros(size) self._tup[0] = self._tup[1] = np.NaN self._tdn[0] = self._tdn[1] = np.NaN for i in range(2, size): if bbe[i - 1] > bbe[i] and bbe[i - 2] < bbe[i - 1]: self._tup[i] = bbe[i] else: self._tup[i] = np.NaN for i in range(2, size): if bbe[i - 1] < bbe[i] and bbe[i - 2] > bbe[i - 1]: self._tdn[i] = bbe[i] else: self._tdn[i] = np.NaN highest = ta_MAX(high, 3) lowest = ta_MIN(low, 3) last_up = 0.0 last_dn = 0.0 for i in range(2, size): if not np.isnan(self._tup[i]): last_up = self._tup[i] = highest[i] elif last_up > 0.0: self._tup[i] = last_up if not np.isnan(self._tdn[i]): last_dn = self._tdn[i] = lowest[i] elif last_dn > 0.0: self._tdn[i] = last_dn # logger.debug("%s %s %s" % (self._tdn[-3], self._tdn[-2], self._tdn[-1])) # logger.debug("%s - %s %s / %s %s %s / %s %s %s / %s %s %s / %s %s %s" % ( # datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S'), # datetime.utcfromtimestamp(timestamps[-2]).strftime('%Y-%m-%d %H:%M:%S'), # datetime.utcfromtimestamp(timestamps[-1]).strftime('%Y-%m-%d %H:%M:%S'), # self._tdn[-3], self._tdn[-2], self._tdn[-1], close[-3], close[-2], close[-1], low[-3], low[-2], low[-1], lowest[-3], lowest[-2], lowest[-1])) # compact timeserie from_timestamp = Instrument.basetime(self.timeframe, self._last_timestamp) # inclusive to_timestamp = Instrument.basetime(self.timeframe, timestamp) # exclusive delta = min( int((to_timestamp - from_timestamp) / self._timeframe) + 1, len(timestamps)) # base index num = len(timestamps) last_up = self._up[-1] if len(self._up) else np.NaN last_dn = self._down[-1] if len(self._down) else np.NaN # if len(self._tdn) and not np.isnan(self._tdn[-1]) and self._tdn[-1] != last_dn: # self._down.append(self._tdn[-1]) # last_dn = self._tdn[-1] # logger.info("> %s %s" % (datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S'), last_dn)) # if len(self._tup) and not np.isnan(self._tup[-1]) and self._tup[-1] != last_up: # self._up.append(self._tup[-1]) for b in range(num - delta, num): # only most recent and complete if from_timestamp <= timestamps[b] < to_timestamp: if not np.isnan( self._tup[b] ) and self._tup[b] != last_up and self._tup[b] > 0.0: last_up = self._tup[b] self._up.append(last_up) self._both.append(last_up) if len(self._up) > self._max_history: self._up.pop(0) if len(self._both) > 2 * self._max_history: self._both.pop(0) if not np.isnan( self._tdn[b] ) and self._tdn[b] != last_dn and self._tdn[b] > 0.0: last_dn = self._tdn[b] # logger.info("%s %s" % (datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S'), last_dn)) self._down.append(last_dn) self._both.append(last_dn) if len(self._down) > self._max_history: self._down.pop(0) if len(self._both) > 2 * self._max_history: self._both.pop(0) # if self.timeframe == 60: # logger.info("%s %s" % (self._tup, self._tdn)) # logger.info("%s %s" % (self._up, self._down)) # if self.timeframe == 60*60*24: # logger.info("%s" % (self._both)) if len(atrs): # retains last ATR value self._last_atr = atrs[-1] self._last_timestamp = timestamp return self._up, self._down