def with_custom_contributors():
# Clean table contributor before adding any elements as we fill two contributors in the function clean_db
db.session.execute("TRUNCATE table contributor CASCADE;")
db.session.commit()
db.session.add_all([
model.Contributor("realtime.sherbrooke", "ca",
ConnectorType.gtfs_rt.value, "my_token",
"http://feed.url", 5),
model.Contributor("realtime.paris", "idf", ConnectorType.gtfs_rt.value,
"my_other_token", "http://otherfeed.url"),
model.Contributor("realtime.london", "gb", ConnectorType.cots.value),
])
db.session.commit()
def test_multiple_delays(setup_database, navitia_vj):
"""
We receive a delay on the first and second stoptimes of a vj, and there was already some delay on the
first st of this vj
sa:1 sa:2 sa:3
VJ navitia 8:10 9:05-9:10 10:05
VJ in db 8:15* 9:05-9:10 10:05
update kirin 8:20* *9:07-9:10 10:05
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
# Note: the delay is based of the navitia's vj
StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=10), dep_status="update"),
StopTimeUpdate({"id": "sa:2"}, arrival_delay=timedelta(minutes=2), arr_status="update"),
]
res, _ = handle(builder, real_time_update, [trip_update])
_check_multiples_delay(res)
# we also check that there is what we want in the db
db_trip_updates = res.query.from_self(TripUpdate).all()
assert len(db_trip_updates) == 2
for tu in db_trip_updates:
assert tu.status == "update"
assert len(RealTimeUpdate.query.all()) == 3 # 2 already in db, one new update
assert len(StopTimeUpdate.query.all()) == 6 # 3 st * 2 vj in the db
def test_multiple_delays_in_2_updates(navitia_vj):
"""
same test as test_multiple_delays, but with nothing in the db and with 2 trip updates
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update")
]
handle(builder, real_time_update, [trip_update])
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=10), dep_status="update"),
StopTimeUpdate({"id": "sa:2"}, arrival_delay=timedelta(minutes=2), arr_status="update"),
]
res, _ = handle(builder, real_time_update, [trip_update])
_check_multiples_delay(res)
# we also check that there is what we want in the db
db_trip_updates = res.query.from_self(TripUpdate).all()
assert len(db_trip_updates) == 1
assert db_trip_updates[0].status == "update"
assert len(RealTimeUpdate.query.all()) == 2
assert len(StopTimeUpdate.query.all()) == 3
def test_get_action_on_trip_delete(mock_navitia_fixture):
with app.app_context():
# Delete the recently added trip followed by add: should be FIRST_TIME_ADDED
contributor = model.Contributor(
id=COTS_CONTRIBUTOR_ID,
navitia_coverage=None,
connector_type=ConnectorType.cots.value)
builder = KirinModelBuilder(contributor)
input_trip_add = get_fixture_data("cots_train_151515_added_trip.json")
wrap_build(builder, input_trip_add)
input_trip_delete = get_fixture_data(
"cots_train_151515_deleted_trip_with_delay_and_stop_time_added.json"
)
wrap_build(builder, input_trip_delete)
input_added_trip = get_fixture_data(
"cots_train_151515_added_trip.json")
json_data = json.loads(input_added_trip)
dict_version = model_maker.get_value(json_data, "nouvelleVersion")
train_numbers = model_maker.get_value(dict_version, "numeroCourse")
pdps = model_maker._retrieve_interesting_pdp(
model_maker.get_value(dict_version, "listePointDeParcours"))
action_on_trip = model_maker._get_action_on_trip(
train_numbers, dict_version, pdps)
assert action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name
def test_delays_then_cancellation(setup_database, navitia_vj):
"""
We have a delay on the first st of a vj in the db and we receive a cancellation on this vj,
we should have a cancelled vj in the end
sa:1 sa:2 sa:3
VJ navitia 8:10 9:05-9:10 10:05
VJ in db 8:15* 9:05-9:10 10:05
update kirin -
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="delete",
effect=TripEffect.NO_SERVICE.name,
contributor_id=contributor.id,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "delete"
assert len(trip_update.stop_time_updates) == 3
for stu in trip_update.stop_time_updates:
assert stu.arrival_status == "delete"
assert stu.departure_status == "delete"
assert len(trip_update.real_time_updates) == 2
def test_cots_train_trip_removal(mock_navitia_fixture):
"""
test the import of cots_train_6113_trip_removal.json
"""
input_train_trip_removed = get_fixture_data(
"cots_train_6113_trip_removal.json")
with app.app_context():
contributor = model.Contributor(
id=COTS_CONTRIBUTOR_ID,
navitia_coverage=None,
connector_type=ConnectorType.cots.value)
wrap_build(KirinModelBuilder(contributor), input_train_trip_removed)
trip_updates = TripUpdate.query.all()
assert len(trip_updates) == 1
trip_up = trip_updates[0]
assert trip_up.vj.navitia_trip_id == "trip:OCETGV-87686006-87751008-2:25768"
assert trip_up.vj_id == trip_up.vj.id
assert trip_up.status == "delete"
# full trip removal : no stop_time to precise
assert len(trip_up.stop_time_updates) == 0
# verify trip_update effect:
assert trip_up.effect == "NO_SERVICE"
def test_delays_then_cancellation_in_2_updates(navitia_vj):
"""
Same test as above, but with nothing in the db, and with 2 updates
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update")
]
handle(builder, real_time_update, [trip_update])
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="delete",
effect=TripEffect.NO_SERVICE.name,
contributor_id=contributor.id,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "delete"
assert len(trip_update.stop_time_updates) == 3
for stu in trip_update.stop_time_updates:
assert stu.arrival_status == "delete"
assert stu.departure_status == "delete"
assert len(trip_update.real_time_updates) == 2
def test_put_contributor_with_id(test_client):
db.session.add(
model.Contributor(
id="SaintMeuMeu",
navitia_coverage="ca",
connector_type=ConnectorType.cots.value,
navitia_token="this_is_a_token",
feed_url="http://feed.url",
))
db.session.commit()
resp = test_client.put(
"/contributors/SaintMeuMeu",
json={
"navitia_coverage": "qb",
"connector_type": ConnectorType.gtfs_rt.value,
"navitia_token": "new_token",
"feed_url": "http://new.feed",
"retrieval_interval": 50,
},
)
assert resp.status_code == 200
contrib = db.session.query(model.Contributor).filter(
model.Contributor.id == "SaintMeuMeu").first()
assert contrib.navitia_coverage == "qb"
assert contrib.connector_type == ConnectorType.gtfs_rt.value
assert contrib.navitia_token == "new_token"
assert contrib.feed_url == "http://new.feed"
assert contrib.retrieval_interval == 50
def test_cancellation_then_delay_in_2_updates(navitia_vj):
"""
same as test_cancellation_then_delay, but with a clear db and in 2 updates
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="delete",
effect=TripEffect.NO_SERVICE.name,
contributor_id=contributor.id,
)
trip_update.stop_time_updates = []
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
handle(builder, real_time_update, [trip_update])
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="none",
effect=TripEffect.UNKNOWN_EFFECT.name,
contributor_id=contributor.id,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"}, arr_status="none", dep_status="none", order=0),
StopTimeUpdate({"id": "sa:2"}, arr_status="none", dep_status="none", order=1),
StopTimeUpdate({"id": "sa:3"}, arrival_delay=timedelta(minutes=40), arr_status="update", order=2),
]
res, _ = handle(builder, real_time_update, [trip_update])
_check_cancellation_then_delay(res)
def get_cots_contributor(include_deactivated=False):
"""
:return 1 COTS contributor from config file or db
File has priority over db
TODO: Remove from config file
"""
if "COTS_CONTRIBUTOR" in current_app.config and current_app.config.get(
str("COTS_CONTRIBUTOR")):
return model.Contributor(
id=current_app.config.get(str("COTS_CONTRIBUTOR")),
navitia_coverage=current_app.config.get(str("NAVITIA_INSTANCE")),
connector_type=ConnectorType.cots.value,
navitia_token=current_app.config.get(str("NAVITIA_TOKEN")),
)
else:
contributor = model.Contributor.find_by_connector_type(
ConnectorType.cots.value, include_deactivated=include_deactivated)
if len(contributor) == 0:
logging.getLogger(__name__).error("No COTS contributor found")
raise SubServiceError
if len(contributor) > 1:
logging.getLogger(__name__).warning(
"{n} COTS contributors found in db - {id} taken into account ".
format(n=len(contributor), id=contributor[0].id))
return contributor[0]
def test_past_midnight():
"""
integration of a past midnight
"""
navitia_vj = {
"trip": {"id": "vehicle_journey:1"},
"stop_times": [
{
"utc_arrival_time": datetime.time(22, 10),
"utc_departure_time": datetime.time(22, 15),
"stop_point": {"id": "sa:1"},
},
# arrive at sa:2 at 23:10 and leave the day after
{
"utc_arrival_time": datetime.time(23, 10),
"utc_departure_time": datetime.time(2, 15),
"stop_point": {"id": "sa:2", "stop_area": {"timezone": "UTC"}},
},
{
"utc_arrival_time": datetime.time(3, 20),
"utc_departure_time": datetime.time(3, 25),
"stop_point": {"id": "sa:3", "stop_area": {"timezone": "UTC"}},
},
],
}
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
vj = VehicleJourney(
navitia_vj, datetime.datetime(2015, 9, 8, 21, 15, 0), datetime.datetime(2015, 9, 9, 4, 20, 0)
)
trip_update = TripUpdate(vj, status="update", contributor_id=contributor.id)
st = StopTimeUpdate({"id": "sa:2"}, departure_delay=timedelta(minutes=31), dep_status="update", order=1)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.stop_time_updates) == 3
assert trip_update.stop_time_updates[0].stop_id == "sa:1"
assert trip_update.stop_time_updates[0].arrival == _dt("22:10")
assert trip_update.stop_time_updates[0].departure == _dt("22:15")
assert trip_update.stop_time_updates[1].stop_id == "sa:2"
assert trip_update.stop_time_updates[1].arrival == _dt("23:10")
assert trip_update.stop_time_updates[1].arrival_delay == timedelta(0)
assert trip_update.stop_time_updates[1].departure == _dt("2:46", day=9)
assert trip_update.stop_time_updates[1].departure_delay == timedelta(minutes=31)
assert trip_update.stop_time_updates[2].stop_id == "sa:3"
assert trip_update.stop_time_updates[2].arrival == _dt("3:51", day=9)
assert trip_update.stop_time_updates[2].arrival_delay == timedelta(minutes=31)
assert trip_update.stop_time_updates[2].departure == _dt("3:56", day=9)
assert trip_update.stop_time_updates[2].departure_delay == timedelta(minutes=31)
def get_gtfsrt_contributors(include_deactivated=False):
"""
:return: all GTFS-RT contributors from config file + db
File has priority over db
TODO: Remove from config file
"""
contributor_legacy_id = None
gtfsrt_contributors = []
if "GTFS_RT_CONTRIBUTOR" in current_app.config and current_app.config.get(str("GTFS_RT_CONTRIBUTOR")):
contributor_legacy_id = current_app.config.get(str("GTFS_RT_CONTRIBUTOR"))
contributor_legacy = model.Contributor(
id=contributor_legacy_id,
navitia_coverage=current_app.config.get(str("NAVITIA_GTFS_RT_INSTANCE")),
connector_type=ConnectorType.gtfs_rt.value,
navitia_token=current_app.config.get(str("NAVITIA_GTFS_RT_TOKEN")),
feed_url=current_app.config.get(str("GTFS_RT_FEED_URL")),
)
gtfsrt_contributors.append(contributor_legacy)
gtfsrt_contributors.extend(
[
c
for c in model.Contributor.find_by_connector_type(
ConnectorType.gtfs_rt.value, include_deactivated=include_deactivated
)
if c.id != contributor_legacy_id
]
)
return gtfsrt_contributors
def test_handle_new_vj():
"""an easy one: we have one vj with only one stop time updated"""
navitia_vj = {
"trip": {"id": "vehicle_journey:1"},
"stop_times": [
{
"utc_arrival_time": None,
"utc_departure_time": datetime.time(8, 10),
"stop_point": {"id": "sa:1", "stop_area": {"timezone": "UTC"}},
},
{
"utc_arrival_time": datetime.time(9, 10),
"utc_departure_time": None,
"stop_point": {"id": "sa:2", "stop_area": {"timezone": "UTC"}},
},
],
}
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), contributor_id=contributor.id, status="update")
st = StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update")
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.stop_time_updates) == 2
assert trip_update.stop_time_updates[0].stop_id == "sa:1"
assert trip_update.stop_time_updates[0].departure == _dt("8:15")
assert trip_update.stop_time_updates[0].arrival == _dt("8:15")
assert trip_update.stop_time_updates[1].stop_id == "sa:2"
assert trip_update.stop_time_updates[1].departure == _dt("9:15")
assert trip_update.stop_time_updates[1].arrival == _dt("9:15")
# testing that RealTimeUpdate is persisted in db
db_trip_updates = real_time_update.query.from_self(TripUpdate).all()
assert len(db_trip_updates) == 1
assert db_trip_updates[0].status == "update"
db_st_updates = real_time_update.query.from_self(StopTimeUpdate).order_by("stop_id").all()
assert len(db_st_updates) == 2
assert db_st_updates[0].stop_id == "sa:1"
assert db_st_updates[0].departure == _dt("8:15")
assert db_st_updates[0].arrival == _dt("8:15")
assert db_st_updates[0].trip_update_id == db_trip_updates[0].vj_id
assert db_st_updates[1].stop_id == "sa:2"
assert db_st_updates[1].departure == _dt("9:15")
assert db_st_updates[1].arrival == _dt("9:15")
assert db_st_updates[1].trip_update_id == db_trip_updates[0].vj_id
def test_simple_delay(navitia_vj):
"""Test on delay when there is nothing in the db"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
st = StopTimeUpdate(
{"id": "sa:1"},
arrival_delay=timedelta(minutes=5),
arr_status="update",
departure_delay=timedelta(minutes=10),
dep_status="update",
order=0,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.stop_time_updates) == 3
assert trip_update.stop_time_updates[0].stop_id == "sa:1"
assert trip_update.stop_time_updates[0].departure == _dt("8:20") # 8:10 + 10mn
assert trip_update.stop_time_updates[0].departure_delay == timedelta(minutes=10)
assert trip_update.stop_time_updates[0].departure_status == "update"
assert trip_update.stop_time_updates[1].stop_id == "sa:2"
assert trip_update.stop_time_updates[1].arrival == _dt("9:15")
assert trip_update.stop_time_updates[1].arrival_delay == timedelta(minutes=10)
assert trip_update.stop_time_updates[1].arrival_status == "update"
assert trip_update.stop_time_updates[1].departure == _dt("9:20")
assert trip_update.stop_time_updates[1].departure_delay == timedelta(minutes=10)
assert trip_update.stop_time_updates[1].departure_status == "update"
# testing that RealTimeUpdate is persisted in db
db_trip_updates = real_time_update.query.from_self(TripUpdate).all()
assert len(db_trip_updates) == 1
assert db_trip_updates[0].status == "update"
db_st_updates = real_time_update.query.from_self(StopTimeUpdate).order_by("stop_id").all()
assert len(db_st_updates) == 3
assert db_st_updates[0].stop_id == "sa:1"
assert db_st_updates[0].departure == _dt("8:20") # 8:10 + 10mn
assert db_st_updates[0].departure_delay == timedelta(minutes=10)
assert db_st_updates[0].departure_status == "update"
assert db_st_updates[1].stop_id == "sa:2"
assert db_st_updates[1].arrival == _dt("9:15")
assert db_st_updates[1].departure == _dt("9:20")
assert db_st_updates[1].trip_update_id == db_trip_updates[0].vj_id
assert db_st_updates[2].stop_id == "sa:3"
def post(self, contributor_id=None):
data = flask.request.get_json()
if data is None:
abort(400, message="No posted Json data to create a contributor")
try:
jsonschema.validate(data, self.post_data_schema)
except jsonschema.exceptions.ValidationError as e:
abort(400,
message="Failed to validate posted Json data. Error: {}".
format(e))
contributor_id = contributor_id or data.get("id")
token = data.get("navitia_token", None)
feed_url = data.get("feed_url", None)
retrieval_interval = data.get("retrieval_interval", 10)
is_active = data.get("is_active", True)
broker_url = data.get("broker_url", None)
exchange_name = data.get("exchange_name", None)
queue_name = data.get("queue_name", None)
nb_days_to_keep_trip_update = data.get("nb_days_to_keep_trip_update")
nb_days_to_keep_rt_update = data.get("nb_days_to_keep_rt_update")
if any([broker_url, exchange_name, queue_name
]) and not all([broker_url, exchange_name, queue_name]):
abort(
400,
message=
"'broker_url', 'exchange_name' and 'queue_name' must all be provided or none of them.",
)
try:
new_contrib = model.Contributor(
contributor_id,
data["navitia_coverage"],
data["connector_type"],
token,
feed_url,
retrieval_interval,
is_active,
broker_url,
exchange_name,
queue_name,
nb_days_to_keep_trip_update,
nb_days_to_keep_rt_update,
)
db_commit(new_contrib)
return {"contributor": new_contrib}, 201
except KeyError as e:
err_msg = "Missing attribute '{}' in input data to construct a contributor".format(
e)
abort(400, message=err_msg)
except sqlalchemy.exc.SQLAlchemyError as e:
abort(400,
message="Error while creating contributor - {}".format(e))
def test_put_contributor_with_no_data(test_client):
db.session.add(
model.Contributor(
id="SaintMeuMeu",
navitia_coverage="ca",
connector_type=ConnectorType.cots.value,
navitia_token="this_is_a_token",
feed_url="http://feed.url",
))
db.session.commit()
resp = test_client.put("/contributors/SaintMeuMeu")
assert resp.status_code == 400
def test_manage_consistency(navitia_vj):
"""
we receive an update for a vj already in the database
sa:1 sa:2 sa:3
VJ navitia 08:10 09:05-09:10 10:05
update kirin - *10:15-09:20* -
expected result 08:10-08:10 10:15-10:15 10:15-10:15
> propagating 10 min from sa:2 departure to sa:3, not needing correction for consistency
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
st = StopTimeUpdate(
{"id": "sa:2"},
arrival_delay=timedelta(minutes=70),
dep_status="update",
departure_delay=timedelta(minutes=10),
arr_status="update",
order=1,
)
st.arrival_status = st.departure_status = "update"
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
real_time_update.id = "30866ce8-0638-4fa1-8556-1ddfa22d09d3"
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.real_time_updates) == 1
assert len(trip_update.stop_time_updates) == 3
stu_map = {stu.stop_id: stu for stu in trip_update.stop_time_updates}
assert "sa:1" in stu_map
assert stu_map["sa:1"].arrival == _dt("8:10")
assert stu_map["sa:1"].departure == _dt("8:10")
assert "sa:2" in stu_map
assert stu_map["sa:2"].arrival == _dt("10:15")
assert stu_map["sa:2"].departure == _dt("10:15")
assert "sa:3" in stu_map
assert stu_map["sa:3"].arrival == _dt("10:15")
assert stu_map["sa:3"].departure == _dt("10:15")
def test_handle_basic():
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
with pytest.raises(TypeError):
handle(builder, None, [])
# a RealTimeUpdate without any TripUpdate doesn't do anything
real_time_update = make_rt_update(raw_data=None, contributor_id=GTFS_CONTRIBUTOR_ID)
res, _ = handle(builder, real_time_update, [])
assert res == real_time_update
def clean_db():
"""
before all tests the database is cleared
"""
with app.app_context():
tables = [six.text_type(table) for table in db.metadata.sorted_tables]
db.session.execute("TRUNCATE {} CASCADE;".format(", ".join(tables)))
db.session.commit()
# Add two contributors of each in the table
db.session.add_all([
model.Contributor(
COTS_CONTRIBUTOR_ID,
"sncf",
ConnectorType.cots.value,
"cots_token",
"cots_feed_url",
10,
True,
),
model.Contributor(
PIV_CONTRIBUTOR_ID,
"sncf_piv",
ConnectorType.piv.value,
"piv_token",
"piv_feed_url",
10,
True,
),
model.Contributor(
GTFS_CONTRIBUTOR_ID,
"sherbrooke",
ConnectorType.gtfs_rt.value,
"gtfs-rt_token",
"gtfs-rt_feed_url",
1,
True,
),
model.Contributor(COTS_CONTRIBUTOR_DB_ID, "idfm",
ConnectorType.cots.value, "cots_db_token",
"cots_db_feed_url"),
model.Contributor(PIV_CONTRIBUTOR_DB_ID, "tn",
ConnectorType.piv.value, "piv_db_token",
"piv_db_feed_url"),
model.Contributor(
GTFS_CONTRIBUTOR_DB_ID,
"laval",
ConnectorType.gtfs_rt.value,
"gtfs-rt_db_token",
"gtfs-rt_db_feed_url",
30,
),
])
db.session.commit()
def test_put_partial_contributor(test_client):
db_commit(
model.Contributor(
id="SaintMeuMeu",
navitia_coverage="ca",
connector_type=ConnectorType.cots.value,
navitia_token="this_is_a_token",
feed_url="http://feed.url",
)
)
put_resp = test_client.put("/contributors/SaintMeuMeu", json={"navitia_coverage": "qb"})
assert put_resp.status_code == 200
get_resp = test_client.get("/contributors/SaintMeuMeu")
put_data = json.loads(put_resp.data)
get_data = json.loads(get_resp.data)
assert put_data["contributor"] == get_data["contributors"][0]
def test_cancellation_then_delay(navitia_vj):
"""
we have a cancelled vj in the db, and we receive an update on the 3rd stoptime,
at the end we have a delayed vj
sa:1 sa:2 sa:3
VJ navitia 8:10 9:05-9:10 10:05
VJ in db -
update kirin 8:10 9:05-9:10 10:45*
"""
with app.app_context():
vju = create_trip_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d3",
"vehicle_journey:1",
datetime.date(2015, 9, 8),
[],
status="delete",
)
rtu = make_rt_update(None, contributor_id=GTFS_CONTRIBUTOR_ID)
rtu.id = "10866ce8-0638-4fa1-8556-1ddfa22d09d3"
rtu.trip_updates.append(vju)
db_commit(rtu)
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="update",
effect=TripEffect.UNKNOWN_EFFECT.name,
contributor_id=contributor.id,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:3"}, arrival_delay=timedelta(minutes=40), arr_status="update", order=2)
]
res, _ = handle(builder, real_time_update, [trip_update])
_check_cancellation_then_delay(res)
def test_get_action_on_trip_previously_added(mock_navitia_fixture):
with app.app_context():
# Test for add followed by update should be PREVIOUSLY_ADDED
input_trip_add = get_fixture_data("cots_train_151515_added_trip.json")
contributor = model.Contributor(
id=COTS_CONTRIBUTOR_ID,
navitia_coverage=None,
connector_type=ConnectorType.cots.value)
builder = KirinModelBuilder(contributor)
wrap_build(builder, input_trip_add)
input_update_added_trip = get_fixture_data(
"cots_train_151515_added_trip_with_delay.json")
json_data = json.loads(input_update_added_trip)
dict_version = model_maker.get_value(json_data, "nouvelleVersion")
train_numbers = model_maker.get_value(dict_version, "numeroCourse")
pdps = model_maker._retrieve_interesting_pdp(
model_maker.get_value(dict_version, "listePointDeParcours"))
action_on_trip = model_maker._get_action_on_trip(
train_numbers, dict_version, pdps)
assert action_on_trip == ActionOnTrip.PREVIOUSLY_ADDED.name
def test_handle_new_trip_out_of_order(navitia_vj):
"""
We have one vj with only one stop time updated, but it's not the first
so we have to reorder the stop times in the resulting trip_update
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
st = StopTimeUpdate(
{"id": "sa:2"},
arrival_delay=timedelta(minutes=44),
arr_status="update",
departure_delay=timedelta(minutes=40),
dep_status="update",
order=1,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert len(trip_update.stop_time_updates) == 3
assert trip_update.stop_time_updates[0].stop_id == "sa:1"
assert trip_update.stop_time_updates[0].arrival == _dt("8:10")
assert trip_update.stop_time_updates[0].departure == _dt("8:10")
assert trip_update.stop_time_updates[1].stop_id == "sa:2"
assert trip_update.stop_time_updates[1].arrival == _dt("9:49")
assert trip_update.stop_time_updates[1].departure == _dt("9:50")
assert trip_update.stop_time_updates[2].stop_id == "sa:3"
assert trip_update.stop_time_updates[2].arrival == _dt("10:45")
assert trip_update.stop_time_updates[2].departure == _dt("10:45")
def test_get_action_on_trip_add(mock_navitia_fixture):
"""
Test the function _get_action_on_trip with different type of flux cots
returns:
1. Fist trip add(AJOUTEE)-> FIRST_TIME_ADDED
2. Add followed by update (PERTURBEE) -> PREVIOUSLY_ADDED
3. Delete followed by add -> FIRST_TIME_ADDED
"""
with app.app_context():
# Test for the first add: should be FIRST_TIME_ADDED
input_trip_add = get_fixture_data("cots_train_151515_added_trip.json")
json_data = json.loads(input_trip_add)
dict_version = model_maker.get_value(json_data, "nouvelleVersion")
train_numbers = model_maker.get_value(dict_version, "numeroCourse")
pdps = model_maker._retrieve_interesting_pdp(
model_maker.get_value(dict_version, "listePointDeParcours"))
action_on_trip = model_maker._get_action_on_trip(
train_numbers, dict_version, pdps)
assert action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name
# Test for add followed by update should be PREVIOUSLY_ADDED
contributor = model.Contributor(
id=COTS_CONTRIBUTOR_ID,
navitia_coverage=None,
connector_type=ConnectorType.cots.value)
builder = KirinModelBuilder(contributor)
wrap_build(builder, input_trip_add)
input_update_added_trip = get_fixture_data(
"cots_train_151515_added_trip_with_delay.json")
json_data = json.loads(input_update_added_trip)
dict_version = model_maker.get_value(json_data, "nouvelleVersion")
train_numbers = model_maker.get_value(dict_version, "numeroCourse")
pdps = model_maker._retrieve_interesting_pdp(
model_maker.get_value(dict_version, "listePointDeParcours"))
action_on_trip = model_maker._get_action_on_trip(
train_numbers, dict_version, pdps)
assert action_on_trip == ActionOnTrip.PREVIOUSLY_ADDED.name
# Clean database for further test
clean_db()
# Delete the recently added trip followed by add: should be FIRST_TIME_ADDED
wrap_build(builder, input_trip_add)
input_trip_delete = get_fixture_data(
"cots_train_151515_deleted_trip_with_delay_and_stop_time_added.json"
)
wrap_build(builder, input_trip_delete)
input_added_trip = get_fixture_data(
"cots_train_151515_added_trip.json")
json_data = json.loads(input_added_trip)
dict_version = model_maker.get_value(json_data, "nouvelleVersion")
train_numbers = model_maker.get_value(dict_version, "numeroCourse")
pdps = model_maker._retrieve_interesting_pdp(
model_maker.get_value(dict_version, "listePointDeParcours"))
action_on_trip = model_maker._get_action_on_trip(
train_numbers, dict_version, pdps)
assert action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name
def clean_db(rabbitmq_docker_fixture):
"""
before all tests the database is cleared
"""
with app.app_context():
tables = [str(table) for table in db.metadata.sorted_tables]
db.session.execute("TRUNCATE {} CASCADE;".format(", ".join(tables)))
db.session.commit()
# Add two contributors of each in the table
db.session.add_all(
[
model.Contributor(
COTS_CONTRIBUTOR_ID,
"sncf",
ConnectorType.cots.value,
"cots_token",
"cots_feed_url",
10,
True,
),
model.Contributor(
PIV_CONTRIBUTOR_ID,
"sncf_piv",
ConnectorType.piv.value,
"piv_token",
None,
None,
True,
rabbitmq_docker_fixture.url,
PIV_EXCHANGE_NAME,
PIV_QUEUE_NAME,
),
model.Contributor(
GTFS_CONTRIBUTOR_ID,
"sherbrooke",
ConnectorType.gtfs_rt.value,
"gtfs-rt_token",
"gtfs-rt_feed_url",
1,
True,
),
model.Contributor(
SIRI_ET_LITE_IDFM_CONTRIBUTOR_ID,
"idfm",
ConnectorType.siri_et_lite_idfm.value,
"siri-et-lite-idfm-token",
"siri-et-lite-idfm-feed-url",
1,
True,
),
model.Contributor(
SIRI_ET_XML_TN_CONTRIBUTOR_ID,
"transilien",
ConnectorType.siri_et_xml_tn.value,
"siri_et_xml_tn_db_token",
None,
None,
True,
rabbitmq_docker_fixture.url,
SIRI_ET_XML_TN_EXCHANGE_NAME,
SIRI_ET_XML_TN_QUEUE_NAME,
),
model.Contributor(
COTS_CONTRIBUTOR_DB_ID, "idfm", ConnectorType.cots.value, "cots_db_token", "cots_db_feed_url"
),
model.Contributor(
PIV_CONTRIBUTOR_DB_ID,
"sncf",
ConnectorType.piv.value,
"piv_db_token",
None,
None,
True,
rabbitmq_docker_fixture.url,
PIV_EXCHANGE_NAME,
PIV_QUEUE_NAME,
),
model.Contributor(
SIRI_ET_XML_TN_CONTRIBUTOR_DB_ID,
"transilien",
ConnectorType.siri_et_xml_tn.value,
"siri_et_xml_tn_db_token",
None,
None,
True,
rabbitmq_docker_fixture.url,
SIRI_ET_XML_TN_EXCHANGE_NAME,
SIRI_ET_XML_TN_QUEUE_NAME,
),
model.Contributor(
GTFS_CONTRIBUTOR_DB_ID,
"laval",
ConnectorType.gtfs_rt.value,
"gtfs-rt_db_token",
"gtfs-rt_db_feed_url",
30,
),
]
)
db.session.commit()
def test_cots_train_delayed(mock_navitia_fixture):
"""
test the import of cots_train_96231_delayed.json
"""
input_train_delayed = get_fixture_data("cots_train_96231_delayed.json")
with app.app_context():
contributor = model.Contributor(
id=COTS_CONTRIBUTOR_ID,
navitia_coverage=None,
connector_type=ConnectorType.cots.value)
wrap_build(KirinModelBuilder(contributor), input_train_delayed)
trip_updates = TripUpdate.query.all()
assert len(trip_updates) == 1
trip_up = trip_updates[0]
assert trip_up.vj.navitia_trip_id == "trip:OCETrainTER-87212027-85000109-3:11859"
assert trip_up.vj_id == trip_up.vj.id
assert trip_up.status == "update"
assert trip_up.effect == "SIGNIFICANT_DELAYS"
# 5 stop times must have been created
assert len(trip_up.stop_time_updates) == 6
# first impacted stop time should be 'gare de Sélestat'
st = trip_up.stop_time_updates[1]
assert st.id
assert st.stop_id == "stop_point:OCE:SP:TrainTER-87214056"
# the COTS data has no listeHoraireProjeteArrivee, so the status is 'none'
assert st.arrival == datetime(2015, 9, 21, 15, 38, 0)
assert st.arrival_delay == timedelta(minutes=0)
assert st.arrival_status == "none"
assert st.departure == datetime(2015, 9, 21, 15, 55, 0)
assert st.departure_delay == timedelta(minutes=15)
assert st.departure_status == "update"
assert st.message == "Affluence exceptionnelle de voyageurs"
# second impacted should be 'gare de Colmar'
st = trip_up.stop_time_updates[2]
assert st.id
assert st.stop_id == "stop_point:OCE:SP:TrainTER-87182014"
assert st.arrival == datetime(2015, 9, 21, 16, 6, 0)
assert st.arrival_delay == timedelta(minutes=15)
assert st.arrival_status == "update"
assert st.departure == datetime(2015, 9, 21, 16, 8, 0)
assert st.departure_delay == timedelta(minutes=15)
assert st.departure_status == "update"
assert st.message == "Affluence exceptionnelle de voyageurs"
# last should be 'gare de Basel-SBB'
st = trip_up.stop_time_updates[-1]
assert st.id
assert st.stop_id == "stop_point:OCE:SP:TrainTER-85000109"
assert st.arrival == datetime(2015, 9, 21, 16, 54, 0)
assert st.arrival_delay == timedelta(minutes=15)
assert st.arrival_status == "update"
# no departure since it's the last (thus the departure will be before the arrival)
assert st.departure == datetime(2015, 9, 21, 16, 54, 0)
assert st.departure_delay == timedelta(minutes=15)
assert st.departure_status == "none"
assert st.message == "Affluence exceptionnelle de voyageurs"
def test_handle_update_vj(setup_database, navitia_vj):
"""
this time we receive an update for a vj already in the database
sa:1 sa:2 sa:3
VJ navitia 8:10 9:05-9:10 10:05
VJ in db 8:15* 9:05-9:10 10:05
update kirin - *9:15-9:20* -
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
st = StopTimeUpdate(
{"id": "sa:2"},
arrival_delay=timedelta(minutes=10),
dep_status="update",
departure_delay=timedelta(minutes=10),
arr_status="update",
order=1,
)
st.arrival_status = st.departure_status = "update"
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
real_time_update.id = "30866ce8-0638-4fa1-8556-1ddfa22d09d3"
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.real_time_updates) == 2
assert len(trip_update.stop_time_updates) == 3
stu_map = {stu.stop_id: stu for stu in trip_update.stop_time_updates}
assert "sa:1" in stu_map
assert stu_map["sa:1"].arrival == _dt("8:15")
assert stu_map["sa:1"].departure == _dt("8:15")
assert "sa:2" in stu_map
assert stu_map["sa:2"].arrival == _dt("9:15")
assert stu_map["sa:2"].departure == _dt("9:20")
assert "sa:3" in stu_map
assert stu_map["sa:3"].arrival == _dt("10:15")
assert stu_map["sa:3"].departure == _dt("10:15")
# testing that RealTimeUpdate is persisted in db
db_trip_updates = TripUpdate.query.join(VehicleJourney).order_by("start_timestamp").all()
assert len(db_trip_updates) == 2
assert real_time_update.query.from_self(TripUpdate).all()[0].status == "update"
st_updates = real_time_update.query.from_self(StopTimeUpdate).order_by("stop_id").all()
assert len(st_updates) == 6
# testing that trip update on 2015/09/07 is remaining correctly stored in db
assert len(db_trip_updates[0].stop_time_updates) == 3
db_stu_map = {stu.stop_id: stu for stu in db_trip_updates[0].stop_time_updates}
assert "sa:1" in db_stu_map
assert db_stu_map["sa:1"].arrival is None
assert db_stu_map["sa:1"].departure == _dt("8:35", day=7)
assert "sa:2" in db_stu_map
assert db_stu_map["sa:2"].arrival == _dt("9:35", day=7)
assert db_stu_map["sa:2"].departure == _dt("9:40", day=7)
assert "sa:3" in db_stu_map
assert db_stu_map["sa:3"].arrival == _dt("10:35", day=7)
assert db_stu_map["sa:3"].departure is None
# testing that trip update on 2015/09/08 is correctly merged and stored in db
assert len(db_trip_updates[1].stop_time_updates) == 3
db_stu_map = {stu.stop_id: stu for stu in db_trip_updates[1].stop_time_updates}
assert "sa:1" in db_stu_map
assert db_stu_map["sa:1"].arrival == _dt("8:15")
assert db_stu_map["sa:1"].departure == _dt("8:15")
assert "sa:2" in db_stu_map
assert db_stu_map["sa:2"].arrival == _dt("9:15")
assert db_stu_map["sa:2"].departure == _dt("9:20")
assert "sa:3" in db_stu_map
assert db_stu_map["sa:3"].arrival == _dt("10:15")
assert db_stu_map["sa:3"].departure == _dt("10:15")
