def read(self, exchange, dtype, pair): key = f'{dtype}-{exchange}-{pair}' data = self.conn.xread( {key: '0-0' if key not in self.last_id else self.last_id[key]}) if len(data) == 0: return [] LOG.info("%s: Read %d messages from Redis", key, len(data[0][1])) ret = [] for update_id, update in data[0][1]: if dtype in {L2_BOOK, L3_BOOK}: update = json.loads(update['data']) update = book_flatten(update, update['timestamp'], update['delta']) for u in update: for k in ('size', 'amount', 'price', 'timestamp'): if k in u: u[k] = float(u[k]) ret.extend(update) if dtype == TRADES: for k in ('size', 'amount', 'price', 'timestamp'): if k in update: update[k] = float(update[k]) ret.append(update) self.ids[key].append(update_id) self.last_id[key] = self.ids[key][-1] return ret
def read(self, exchange, dtype, pair): key = f'{dtype}-{exchange}-{pair}' data = self._conn(key).consume(1000000, timeout=0.5) LOG.info("%s: Read %d messages from Kafka", key, len(data)) ret = [] for message in data: self.ids[key] = message update = json.loads(message.value().decode('utf8')) if dtype in {L2_BOOK, L3_BOOK}: update = book_flatten(update, update['timestamp'], update['delta']) ret.extend(update) if dtype in {TRADES, TICKER}: ret.append(update) return ret
def read(self, exchange, dtype, pair, start=None, end=None): key = f'{dtype}-{exchange}-{pair}' if start and end: data = [[key, self.conn.xrange(key, min=start, max=end)]] elif self.batch_size: data = [[key, self.conn.xrange(key, count=self.batch_size)]] else: data = self.conn.xread( {key: '0-0' if key not in self.last_id else self.last_id[key]}) if len(data) == 0 or len(data[0][1]) == 0: return [] LOG.info("%s: Read %d messages from Redis", key, len(data[0][1])) ret = [] for update_id, update in data[0][1]: if dtype in {L2_BOOK, L3_BOOK}: update = json.loads(update['data']) update = book_flatten(update, update['timestamp'], update['receipt_timestamp'], update['delta']) for u in update: for k in ('size', 'amount', 'price', 'timestamp', 'receipt_timestamp'): if k in u: u[k] = float(u[k]) ret.extend(update) elif dtype in {TRADES, TICKER, OPEN_INTEREST}: for k in ('size', 'amount', 'price', 'timestamp', 'receipt_timestamp', 'bid', 'ask', 'open_interest'): if k in update: update[k] = float(update[k]) ret.append(update) elif dtype == FUNDING: for k in update: try: update[k] = float(update[k]) except ValueError: # ignore strings pass ret.append(update) self.ids[key].append(update_id) self.last_id[key] = self.ids[key][-1] return ret
def read(self, exchange, dtype, pair, start=None, end=None): kafka = StorageEngines.confluent_kafka key = f'{dtype}-{exchange}-{pair}' if start and end: start_offset = self._conn(key).offsets_for_times( [kafka.TopicPartition(key, 0, start)])[0] stop_offset = self._conn(key).offsets_for_times( [kafka.TopicPartition(key, 0, end)])[0] if start_offset.offset == -1: return [] self._conn(key).assign([start_offset]) offset_diff = stop_offset.offset - start_offset.offset if offset_diff <= 0: return [] data = self._conn(key).consume(offset_diff) self._conn(key).unassign() else: data = self._conn(key).consume(1000000, timeout=0.5) LOG.info("%s: Read %d messages from Kafka", key, len(data)) ret = [] for message in data: self.ids[key] = message msg = message.value().decode('utf8') try: update = json.loads(msg) except Exception: if 'Subscribed topic not available' in msg: return ret if dtype in {L2_BOOK, L3_BOOK}: update = book_flatten(update, update['timestamp'], update['delta'], update['receipt_timestamp']) ret.extend(update) if dtype in {TRADES, TICKER, FUNDING, OPEN_INTEREST}: ret.append(update) return ret