def test_historical_stream(self):
trading = betfairlightweight.APIClient("username",
"password",
app_key="appKey")
stream = trading.streaming.create_historical_stream(
file_path="tests/resources/historicaldata/RACE-1.140075353",
listener=StreamListener(),
operation="raceSubscription",
)
stream.start()
for cache in stream.listener.stream._caches.values():
cache.create_resource(1, False)
assert stream.listener.stream_type == "raceSubscription"
assert stream.listener.stream_unique_id == 0
assert stream.listener.stream._updates_processed == 4
assert len(stream.listener.stream._caches) == 2
market = stream.listener.stream._caches.get("1.1234567")
assert len(market.rrc) == 2
market = stream.listener.stream._caches.get("1.173853449")
assert len(market.rrc) == 4
assert stream._running is False
def __init__(
self,
client: betfairlightweight.APIClient,
market_filter: dict,
market_data_filter: dict,
conflate_ms: int = None,
streaming_unique_id: int = 10000, # 10 seconds
):
threading.Thread.__init__(self, daemon=True, name=self.__class__.__name__)
self.client = client
self.market_filter = market_filter
self.market_data_filter = market_data_filter
self.conflate_ms = conflate_ms
self.streaming_unique_id = streaming_unique_id
self.stream = None
self.output_queue = queue.Queue()
self.listener = StreamListener(output_queue=self.output_queue)
def test_historical_generator_stream(self):
# assert that data is processed correctly (regression testing)
trading = betfairlightweight.APIClient("username",
"password",
app_key="appKey")
stream = trading.streaming.create_historical_generator_stream(
file_path="tests/resources/historicaldata/BASIC-1.132153978",
listener=StreamListener(lightweight=True),
)
gen = stream.get_generator()
data = [i[0] for i in gen()]
with open(
"tests/resources/historicaldata/BASIC-1.132153978-processed.json",
"r") as f:
expected_data = load(f)
assert expected_data == data
def test_historical_stream(self):
trading = betfairlightweight.APIClient('username', 'password', app_key='appKey')
stream = trading.streaming.create_historical_stream(
directory='tests/resources/historicaldata/BASIC-1.132153978',
listener=StreamListener()
)
stream.start(_async=False)
assert stream.listener.stream_type == 'marketSubscription'
assert stream.listener.stream_unique_id == 'HISTORICAL'
assert stream.listener.clk == '3522512789'
assert stream.listener.stream._updates_processed == 480
assert len(stream.listener.stream._caches) == 1
market = stream.listener.stream._caches.get('1.132153978')
assert len(market.runners) == 14
assert stream._running is False
def test_historical_stream(self):
trading = betfairlightweight.APIClient("username",
"password",
app_key="appKey")
stream = trading.streaming.create_historical_stream(
file_path="tests/resources/historicaldata/BASIC-1.132153978",
listener=StreamListener(),
)
stream.start()
assert stream.listener.stream_type == "marketSubscription"
assert stream.listener.stream_unique_id == 0
assert stream.listener.clk == "3522512789"
assert stream.listener.stream._updates_processed == 480
assert len(stream.listener.stream._caches) == 1
market = stream.listener.stream._caches.get("1.132153978")
assert len(market.runners) == 14
assert stream._running is False
def test_historical_stream(self):
trading = betfairlightweight.APIClient("username",
"password",
app_key="appKey")
stream = trading.streaming.create_historical_stream(
file_path="tests/resources/historicaldata/CRICKET-1.179676557",
listener=StreamListener(),
operation="cricketSubscription",
)
stream.start()
for cache in stream.listener.stream._caches.values():
cache.create_resource(1, False)
assert stream.listener.stream_type == "cricketSubscription"
assert stream.listener.stream_unique_id == 0
assert stream.listener.stream._updates_processed == 3
assert len(stream.listener.stream._caches) == 1
market = stream.listener.stream._caches.get("1.179676557")
assert market is not None
assert stream._running is False
psql_dsn = {
'host': 'localhost,s3.vpn.mele.law',
'port': '5432',
'user': '******',
'dbname': 'betfair'
}
db = db.DB(psql_dsn=psql_dsn)
# setup logging
logging.basicConfig(level=logging.INFO)
# create trading instance (don't need username/password)
trading = betfairlightweight.APIClient("username", "password", app_key='')
# create listener
listener = StreamListener(max_latency=None)
def snapshot(_db, _market_book, _seconds_to_start):
_raceid = int(_market_book.market_id.replace('1.1', '11'))
_milliseconds_to_start = _seconds_to_start * 1000
_event_type = int(_market_book.market_definition.event_type_id)
_country_code = _market_book.market_definition.country_code
try:
_d = market_book.json().decode('utf-8')
except:
_d = market_book.json()
if _db.data_exists(_raceid):
logging.info(f'{_raceid} already inserted')
return
basepath = './data/2020/11'
days_list = os.listdir(basepath)
days_list = [i for i in days_list
if '.DS_Store' not in i] # Filter out mac b.s.
# Configure this to be the number of seconds before the jump you want to grab the price at
offset = 600
# Main Execution starts here
files = get_file_names(basepath, days_list)
data = []
for f_name in files[:50]:
print(f_name)
f_pointer = bz2.BZ2File(f_name, 'rb')
listener = StreamListener(max_latency=None, lightweight=True)
listener.register_stream(0, "marketSubscription")
# create historical stream (update directory to your file location)
stream = HistoricalStreamMod(file_stream=f_pointer, listener=listener)
gen = stream.get_generator()
mb = get_book_json(gen)
if mb:
race_data = extract_data_json(mb)
data = data + race_data
df = pd.DataFrame(data,
columns=[
'date', 'track', 'name', 'market_id', 'selection_id',
'selection_name', 'status', 'p1', 'p2', 'p3', 'p4', 'p5',
'v1', 'v2', 'v3', 'v4', 'v5', 'l1', 'l2', 'l3', 'l4',
'l5', 'lv1', 'lv2', 'lv3', 'lv4', 'lv5'