def test_warehouse_updates_queries_source(source, storage):
cfg = {'packet_id': {'storage': "some/directory", 'interval': 30, 'pair': 'SMNPAR',
'source': {'type': "some_source"}},
'other_pkt': {'storage': "some/directory", 'interval': 60, 'pair': 'SMNPAR'}}
warehouse = Warehouse(cfg)
storage.last_time_of("some/directory", interval=30, pair='SMNPAR').set(15000)
warehouse.update('packet_id')
assert source.received_query_since == 15000 + 30
def test_warehouse_updates_source_from_specified_start_time_if_they_do_not_exist_yet(source, storage):
cfg = {'packet_id': {'storage': "some/directory", 'interval': 30, 'pair': 'SMNPAR',
'source': {'type': "some_source"}, 'start': 1000},
'other_pkt': {'storage': "some/directory", 'interval': 60, 'pair': 'SMNPAR'}}
warehouse = Warehouse(cfg)
storage.set_not_existent()
warehouse.update('packet_id')
assert source.received_query_since == 1000
def test_warehouse_retrieves_specified_packet_range():
warehouse = Warehouse(
{'packet_id': {'storage': "some/directory", 'interval': 30, 'pair': 'SMNPAR'},
'other_pkt': {'storage': "some/directory", 'interval': 60, 'pair': 'SMNPAR'}})
assert warehouse.retrieve('packet_id', since=1500000000, until=1500001000) == Data(from_dir="some/directory/SMNPAR",
with_interval=30,
with_since=1500000000,
with_until=1500001000)
def make_warehouse(cfg):
cfg_dict = dict()
for pkt in [p for p in cfg if p != cfg.default_section]:
cfg_dict[pkt] = dict()
for key in cfg[pkt]:
cfg_dict[pkt][key] = cfg[pkt][key]
if key == "exclude_outliers":
cfg_dict[pkt][key] = [
c.strip() for c in cfg_dict[pkt][key].split(',')
]
return Warehouse(cfg_dict)
def test_warehouse_stores_queried_updates(source, storage):
warehouse = Warehouse({'packet_id': {'storage': "some/directory", 'interval': 30, 'pair': 'SMNPAR',
'source': "some_source"},
'other_pkt': {'storage': "some/directory", 'interval': 60, 'pair': 'SMNPAR', }})
warehouse.update('packet_id')
storage.stored(source.returned_datums)
def test_warehouse_passes_validation_config_along_to_query(source):
warehouse = Warehouse({'packet_id': {'storage': "some/directory", 'interval': 30, 'pair': 'SMNPAR',
'source': "some_source", 'exclude_outliers': ['vwap'],
'z_score_threshold': 5}})
warehouse.update('packet_id')
assert source.received_validation_cfg == dict(exclude_outliers=['vwap'], z_score_threshold=5)
def test_warehouse_invokes_configured_query(source):
warehouse = Warehouse({'packet_id': {'storage': "some/directory", 'interval': 30, 'pair': 'SMNPAR',
'source': "some_source"},
'other_pkt': {'storage': "some/directory", 'interval': 60, 'pair': 'SMNPAR', }})
warehouse.update('packet_id')
assert source.updated_from('some_source', with_interval=30, with_pair="SMNPAR", with_storage="some/directory")
def test_warehouse_updating_with_unknown_source():
warehouse = Warehouse({'packet_id': {'storage': "some/directory", 'interval': 30, 'pair': 'SMNPAR',
'source': "some_source"}})
with pytest.raises(NotImplementedError):
warehouse.update('packet_id')
def test_warehouse_reports_all_packets(config, packets):
warehouse = Warehouse(config)
assert warehouse.all_packets() == packets
def test_warehouse_raises_an_error_when_accessing_non_existent_packet():
warehouse = Warehouse({})
with pytest.raises(MissingPacketError):
warehouse.retrieve("some_packet_id")
with pytest.raises(MissingPacketError):
warehouse.update("some_packet_id")