def test_multiple_delays_in_2_updates(navitia_vj):
"""
same test as test_multiple_delays, but with nothing in the db and with 2 trip updates
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update")
]
handle(builder, real_time_update, [trip_update])
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=10), dep_status="update"),
StopTimeUpdate({"id": "sa:2"}, arrival_delay=timedelta(minutes=2), arr_status="update"),
]
res, _ = handle(builder, real_time_update, [trip_update])
_check_multiples_delay(res)
# we also check that there is what we want in the db
db_trip_updates = res.query.from_self(TripUpdate).all()
assert len(db_trip_updates) == 1
assert db_trip_updates[0].status == "update"
assert len(RealTimeUpdate.query.all()) == 2
assert len(StopTimeUpdate.query.all()) == 3
def test_cancellation_then_delay_in_2_updates(navitia_vj):
"""
same as test_cancellation_then_delay, but with a clear db and in 2 updates
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="delete",
effect=TripEffect.NO_SERVICE.name,
contributor_id=contributor.id,
)
trip_update.stop_time_updates = []
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
handle(builder, real_time_update, [trip_update])
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="none",
effect=TripEffect.UNKNOWN_EFFECT.name,
contributor_id=contributor.id,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"}, arr_status="none", dep_status="none", order=0),
StopTimeUpdate({"id": "sa:2"}, arr_status="none", dep_status="none", order=1),
StopTimeUpdate({"id": "sa:3"}, arrival_delay=timedelta(minutes=40), arr_status="update", order=2),
]
res, _ = handle(builder, real_time_update, [trip_update])
_check_cancellation_then_delay(res)
def test_delays_then_cancellation_in_2_updates(navitia_vj):
"""
Same test as above, but with nothing in the db, and with 2 updates
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update")
]
handle(builder, real_time_update, [trip_update])
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="delete",
effect=TripEffect.NO_SERVICE.name,
contributor_id=contributor.id,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "delete"
assert len(trip_update.stop_time_updates) == 3
for stu in trip_update.stop_time_updates:
assert stu.arrival_status == "delete"
assert stu.departure_status == "delete"
assert len(trip_update.real_time_updates) == 2
def test_cancellation_then_delay_in_2_updates(navitia_vj):
"""
same as test_cancellation_then_delay, but with a clear db and in 2 updates
"""
with app.app_context():
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="delete",
contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.stop_time_updates = []
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
handle(real_time_update, [trip_update],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="update",
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:3"},
arrival_delay=timedelta(minutes=40),
arr_status="update",
order=2)
]
res, _ = handle(real_time_update, [trip_update],
COTS_CONTRIBUTOR_ID,
is_new_complete=False)
_check_cancellation_then_delay(res)
def test_past_midnight():
"""
integration of a past midnight
"""
navitia_vj = {
"trip": {"id": "vehicle_journey:1"},
"stop_times": [
{
"utc_arrival_time": datetime.time(22, 10),
"utc_departure_time": datetime.time(22, 15),
"stop_point": {"id": "sa:1"},
},
# arrive at sa:2 at 23:10 and leave the day after
{
"utc_arrival_time": datetime.time(23, 10),
"utc_departure_time": datetime.time(2, 15),
"stop_point": {"id": "sa:2", "stop_area": {"timezone": "UTC"}},
},
{
"utc_arrival_time": datetime.time(3, 20),
"utc_departure_time": datetime.time(3, 25),
"stop_point": {"id": "sa:3", "stop_area": {"timezone": "UTC"}},
},
],
}
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
vj = VehicleJourney(
navitia_vj, datetime.datetime(2015, 9, 8, 21, 15, 0), datetime.datetime(2015, 9, 9, 4, 20, 0)
)
trip_update = TripUpdate(vj, status="update", contributor_id=contributor.id)
st = StopTimeUpdate({"id": "sa:2"}, departure_delay=timedelta(minutes=31), dep_status="update", order=1)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.stop_time_updates) == 3
assert trip_update.stop_time_updates[0].stop_id == "sa:1"
assert trip_update.stop_time_updates[0].arrival == _dt("22:10")
assert trip_update.stop_time_updates[0].departure == _dt("22:15")
assert trip_update.stop_time_updates[1].stop_id == "sa:2"
assert trip_update.stop_time_updates[1].arrival == _dt("23:10")
assert trip_update.stop_time_updates[1].arrival_delay == timedelta(0)
assert trip_update.stop_time_updates[1].departure == _dt("2:46", day=9)
assert trip_update.stop_time_updates[1].departure_delay == timedelta(minutes=31)
assert trip_update.stop_time_updates[2].stop_id == "sa:3"
assert trip_update.stop_time_updates[2].arrival == _dt("3:51", day=9)
assert trip_update.stop_time_updates[2].arrival_delay == timedelta(minutes=31)
assert trip_update.stop_time_updates[2].departure == _dt("3:56", day=9)
assert trip_update.stop_time_updates[2].departure_delay == timedelta(minutes=31)
def build_rt_update(self, input_raw):
# create a raw gtfs-rt obj, save the raw protobuf into the db
proto = gtfs_realtime_pb2.FeedMessage()
log_dict = {}
try:
proto.ParseFromString(input_raw)
except DecodeError:
# We save the non-decodable flux gtfs-rt
rt_update = manage_db_error(
input_raw.encode(
"string_escape",
"ignore"), # protect for PostgreSQL "Text" type
ConnectorType.gtfs_rt.value,
contributor_id=self.contributor.id,
error="invalid protobuf",
is_reprocess_same_data_allowed=False,
)
return rt_update, log_dict
feed = six.binary_type(
proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(
feed,
connector_type=self.contributor.connector_type,
contributor_id=self.contributor.id)
rt_update.proto = proto
return rt_update, log_dict
def handle(proto, navitia_wrapper, contributor):
data = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor)
start_datetime = datetime.datetime.utcnow()
try:
trip_updates = KirinModelBuilder(navitia_wrapper,
contributor).build(rt_update,
data=proto)
record_call('OK', contributor=contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
real_time_update, log_dict = core.handle(rt_update, trip_updates,
contributor)
duration = (datetime.datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({
'duration':
duration,
'input_timestamp':
datetime.datetime.utcfromtimestamp(proto.header.timestamp)
})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def create_real_time_update(id, contributor_id, connector_type, vj_id, trip_id,
circulation_date):
rtu = make_rt_update("", connector_type, contributor_id=contributor_id)
rtu.id = id
trip_update = create_trip_update(vj_id, trip_id, circulation_date)
trip_update.contributor_id = contributor_id
rtu.trip_updates.append(trip_update)
def test_delays_then_cancellation(setup_database, navitia_vj):
"""
We have a delay on the first st of a vj in the db and we receive a cancellation on this vj,
we should have a cancelled vj in the end
sa:1 sa:2 sa:3
VJ navitia 8:10 9:05-9:10 10:05
VJ in db 8:15* 9:05-9:10 10:05
update kirin -
"""
with app.app_context():
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="delete",
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
res, _ = handle(real_time_update, [trip_update],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "delete"
assert len(trip_update.stop_time_updates) == 0
assert len(trip_update.real_time_updates) == 2
def post(self):
raw_xml = get_ire(flask.globals.request)
# create a raw ire obj, save the raw_xml into the db
rt_update = make_rt_update(raw_xml, 'ire')
try:
# assuming UTF-8 encoding for all ire input
rt_update.raw_data = rt_update.raw_data.encode('utf-8')
# raw_xml is interpreted
trip_updates = KirinModelBuilder(self.navitia_wrapper, self.contributor).build(rt_update)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
raise
core.handle(rt_update, trip_updates, current_app.config['CONTRIBUTOR'])
return 'OK', 200
def build_rt_update(self, input_raw):
rt_update = make_rt_update(
input_raw,
connector_type=self.contributor.connector_type,
contributor_id=self.contributor.id)
log_dict = {}
return rt_update, log_dict
def test_delays_then_cancellation(setup_database, navitia_vj):
"""
We have a delay on the first st of a vj in the db and we receive a cancellation on this vj,
we should have a cancelled vj in the end
sa:1 sa:2 sa:3
VJ navitia 8:10 9:05-9:10 10:05
VJ in db 8:15* 9:05-9:10 10:05
update kirin -
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="delete",
effect=TripEffect.NO_SERVICE.name,
contributor_id=contributor.id,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "delete"
assert len(trip_update.stop_time_updates) == 3
for stu in trip_update.stop_time_updates:
assert stu.arrival_status == "delete"
assert stu.departure_status == "delete"
assert len(trip_update.real_time_updates) == 2
def test_multiple_delays(setup_database, navitia_vj):
"""
We receive a delay on the first and second stoptimes of a vj, and there was already some delay on the
first st of this vj
sa:1 sa:2 sa:3
VJ navitia 8:10 9:05-9:10 10:05
VJ in db 8:15* 9:05-9:10 10:05
update kirin 8:20* *9:07-9:10 10:05
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
# Note: the delay is based of the navitia's vj
StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=10), dep_status="update"),
StopTimeUpdate({"id": "sa:2"}, arrival_delay=timedelta(minutes=2), arr_status="update"),
]
res, _ = handle(builder, real_time_update, [trip_update])
_check_multiples_delay(res)
# we also check that there is what we want in the db
db_trip_updates = res.query.from_self(TripUpdate).all()
assert len(db_trip_updates) == 2
for tu in db_trip_updates:
assert tu.status == "update"
assert len(RealTimeUpdate.query.all()) == 3 # 2 already in db, one new update
assert len(StopTimeUpdate.query.all()) == 6 # 3 st * 2 vj in the db
def test_multiple_delays_in_2_updates(navitia_vj):
"""
same test as test_multiple_delays, but with nothing in the db and with 2 trip updates
"""
with app.app_context():
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="update",
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"},
departure_delay=timedelta(minutes=5),
dep_status="update")
]
handle(real_time_update, [trip_update],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="update",
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"},
departure_delay=timedelta(minutes=10),
dep_status="update"),
StopTimeUpdate({"id": "sa:2"},
arrival_delay=timedelta(minutes=2),
arr_status="update"),
]
res, _ = handle(real_time_update, [trip_update],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
_check_multiples_delay(res)
# we also check that there is what we want in the db
db_trip_updates = res.query.from_self(TripUpdate).all()
assert len(db_trip_updates) == 1
assert db_trip_updates[0].status == "update"
assert len(RealTimeUpdate.query.all()) == 2
assert len(StopTimeUpdate.query.all()) == 3
def test_handle_new_vj():
"""an easy one: we have one vj with only one stop time updated"""
navitia_vj = {
"trip": {"id": "vehicle_journey:1"},
"stop_times": [
{
"utc_arrival_time": None,
"utc_departure_time": datetime.time(8, 10),
"stop_point": {"id": "sa:1", "stop_area": {"timezone": "UTC"}},
},
{
"utc_arrival_time": datetime.time(9, 10),
"utc_departure_time": None,
"stop_point": {"id": "sa:2", "stop_area": {"timezone": "UTC"}},
},
],
}
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), contributor_id=contributor.id, status="update")
st = StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update")
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.stop_time_updates) == 2
assert trip_update.stop_time_updates[0].stop_id == "sa:1"
assert trip_update.stop_time_updates[0].departure == _dt("8:15")
assert trip_update.stop_time_updates[0].arrival == _dt("8:15")
assert trip_update.stop_time_updates[1].stop_id == "sa:2"
assert trip_update.stop_time_updates[1].departure == _dt("9:15")
assert trip_update.stop_time_updates[1].arrival == _dt("9:15")
# testing that RealTimeUpdate is persisted in db
db_trip_updates = real_time_update.query.from_self(TripUpdate).all()
assert len(db_trip_updates) == 1
assert db_trip_updates[0].status == "update"
db_st_updates = real_time_update.query.from_self(StopTimeUpdate).order_by("stop_id").all()
assert len(db_st_updates) == 2
assert db_st_updates[0].stop_id == "sa:1"
assert db_st_updates[0].departure == _dt("8:15")
assert db_st_updates[0].arrival == _dt("8:15")
assert db_st_updates[0].trip_update_id == db_trip_updates[0].vj_id
assert db_st_updates[1].stop_id == "sa:2"
assert db_st_updates[1].departure == _dt("9:15")
assert db_st_updates[1].arrival == _dt("9:15")
assert db_st_updates[1].trip_update_id == db_trip_updates[0].vj_id
def test_cancellation_then_delay(navitia_vj):
"""
we have a cancelled vj in the db, and we receive an update on the 3rd stoptime,
at the end we have a delayed vj
sa:1 sa:2 sa:3
VJ navitia 8:10 9:05-9:10 10:05
VJ in db -
update kirin 8:10 9:05-9:10 10:45*
"""
with app.app_context():
vju = create_trip_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d3",
"vehicle_journey:1",
datetime.date(2015, 9, 8),
[],
status="delete",
)
rtu = make_rt_update(None,
ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
rtu.id = "10866ce8-0638-4fa1-8556-1ddfa22d09d3"
rtu.trip_updates.append(vju)
db.session.add(rtu)
db.session.commit()
with app.app_context():
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="update",
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:3"},
arrival_delay=timedelta(minutes=40),
arr_status="update",
order=2)
]
res, _ = handle(real_time_update, [trip_update],
COTS_CONTRIBUTOR_ID,
is_new_complete=False)
_check_cancellation_then_delay(res)
def test_populate_pb_with_cancelation():
"""
VJ cancelation
"""
navitia_vj = {
"trip": {
"id": "vehicle_journey:1"
},
"stop_times": [{
"utc_arrival_time": datetime.time(8, 10),
"stop_point": {
"stop_area": {
"timezone": "UTC"
}
}
}],
}
with app.app_context():
vj = VehicleJourney(navitia_vj,
datetime.datetime(2015, 9, 8, 7, 10, 0),
datetime.datetime(2015, 9, 8, 11, 5, 0))
trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.vj = vj
trip_update.status = "delete"
trip_update.message = "Message Test"
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.company_id = "sncf"
trip_update.effect = "REDUCED_SERVICE"
real_time_update.trip_updates.append(trip_update)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version == "1"
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == "vehicle_journey:1"
assert pb_trip_update.trip.start_date == "20150908"
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is True
assert pb_trip_update.Extensions[
kirin_pb2.trip_message] == "Message Test"
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True
assert pb_trip_update.trip.Extensions[
kirin_pb2.contributor] == COTS_CONTRIBUTOR_ID
assert pb_trip_update.trip.Extensions[kirin_pb2.company_id] == "sncf"
assert pb_trip_update.Extensions[
kirin_pb2.effect] == gtfs_realtime_pb2.Alert.REDUCED_SERVICE
assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_simple_delay(navitia_vj):
"""Test on delay when there is nothing in the db"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
st = StopTimeUpdate(
{"id": "sa:1"},
arrival_delay=timedelta(minutes=5),
arr_status="update",
departure_delay=timedelta(minutes=10),
dep_status="update",
order=0,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.stop_time_updates) == 3
assert trip_update.stop_time_updates[0].stop_id == "sa:1"
assert trip_update.stop_time_updates[0].departure == _dt("8:20") # 8:10 + 10mn
assert trip_update.stop_time_updates[0].departure_delay == timedelta(minutes=10)
assert trip_update.stop_time_updates[0].departure_status == "update"
assert trip_update.stop_time_updates[1].stop_id == "sa:2"
assert trip_update.stop_time_updates[1].arrival == _dt("9:15")
assert trip_update.stop_time_updates[1].arrival_delay == timedelta(minutes=10)
assert trip_update.stop_time_updates[1].arrival_status == "update"
assert trip_update.stop_time_updates[1].departure == _dt("9:20")
assert trip_update.stop_time_updates[1].departure_delay == timedelta(minutes=10)
assert trip_update.stop_time_updates[1].departure_status == "update"
# testing that RealTimeUpdate is persisted in db
db_trip_updates = real_time_update.query.from_self(TripUpdate).all()
assert len(db_trip_updates) == 1
assert db_trip_updates[0].status == "update"
db_st_updates = real_time_update.query.from_self(StopTimeUpdate).order_by("stop_id").all()
assert len(db_st_updates) == 3
assert db_st_updates[0].stop_id == "sa:1"
assert db_st_updates[0].departure == _dt("8:20") # 8:10 + 10mn
assert db_st_updates[0].departure_delay == timedelta(minutes=10)
assert db_st_updates[0].departure_status == "update"
assert db_st_updates[1].stop_id == "sa:2"
assert db_st_updates[1].arrival == _dt("9:15")
assert db_st_updates[1].departure == _dt("9:20")
assert db_st_updates[1].trip_update_id == db_trip_updates[0].vj_id
assert db_st_updates[2].stop_id == "sa:3"
def test_cancellation_then_delay(navitia_vj):
"""
we have a cancelled vj in the db, and we receive an update on the 3rd stoptime,
at the end we have a delayed vj
sa:1 sa:2 sa:3
VJ navitia 8:10 9:05-9:10 10:05
VJ in db -
update kirin 8:10 9:05-9:10 10:45*
"""
with app.app_context():
vju = create_trip_update(
"70866ce8-0638-4fa1-8556-1ddfa22d09d3",
"vehicle_journey:1",
datetime.date(2015, 9, 8),
[],
status="delete",
)
rtu = make_rt_update(None, contributor_id=GTFS_CONTRIBUTOR_ID)
rtu.id = "10866ce8-0638-4fa1-8556-1ddfa22d09d3"
rtu.trip_updates.append(vju)
db_commit(rtu)
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(
_create_db_vj(navitia_vj),
status="update",
effect=TripEffect.UNKNOWN_EFFECT.name,
contributor_id=contributor.id,
)
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:3"}, arrival_delay=timedelta(minutes=40), arr_status="update", order=2)
]
res, _ = handle(builder, real_time_update, [trip_update])
_check_cancellation_then_delay(res)
def test_manage_consistency(navitia_vj):
"""
we receive an update for a vj already in the database
sa:1 sa:2 sa:3
VJ navitia 08:10 09:05-09:10 10:05
update kirin - *10:15-09:20* -
expected result 08:10-08:10 10:15-10:15 10:15-10:15
> propagating 10 min from sa:2 departure to sa:3, not needing correction for consistency
"""
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id)
st = StopTimeUpdate(
{"id": "sa:2"},
arrival_delay=timedelta(minutes=70),
dep_status="update",
departure_delay=timedelta(minutes=10),
arr_status="update",
order=1,
)
st.arrival_status = st.departure_status = "update"
real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id)
real_time_update.id = "30866ce8-0638-4fa1-8556-1ddfa22d09d3"
trip_update.stop_time_updates.append(st)
res, _ = handle(builder, real_time_update, [trip_update])
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.real_time_updates) == 1
assert len(trip_update.stop_time_updates) == 3
stu_map = {stu.stop_id: stu for stu in trip_update.stop_time_updates}
assert "sa:1" in stu_map
assert stu_map["sa:1"].arrival == _dt("8:10")
assert stu_map["sa:1"].departure == _dt("8:10")
assert "sa:2" in stu_map
assert stu_map["sa:2"].arrival == _dt("10:15")
assert stu_map["sa:2"].departure == _dt("10:15")
assert "sa:3" in stu_map
assert stu_map["sa:3"].arrival == _dt("10:15")
assert stu_map["sa:3"].departure == _dt("10:15")
def test_manage_consistency(navitia_vj):
"""
we receive an update for a vj already in the database
sa:1 sa:2 sa:3
VJ navitia 08:10 09:05-09:10 10:05
update kirin - *10:15-09:20* -
expected result 08:10-08:10 10:15-10:15 11:10-11:10
"""
with app.app_context():
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="update",
contributor_id=COTS_CONTRIBUTOR_ID)
st = StopTimeUpdate(
{"id": "sa:2"},
arrival_delay=timedelta(minutes=70),
dep_status="update",
departure_delay=timedelta(minutes=10),
arr_status="update",
order=1,
)
st.arrival_status = st.departure_status = "update"
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update.id = "30866ce8-0638-4fa1-8556-1ddfa22d09d3"
trip_update.stop_time_updates.append(st)
res, _ = handle(real_time_update, [trip_update],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "update"
assert len(trip_update.real_time_updates) == 1
assert len(trip_update.stop_time_updates) == 3
stu_map = {stu.stop_id: stu for stu in trip_update.stop_time_updates}
assert "sa:1" in stu_map
assert stu_map["sa:1"].arrival == _dt("8:10")
assert stu_map["sa:1"].departure == _dt("8:10")
assert "sa:2" in stu_map
assert stu_map["sa:2"].arrival == _dt("10:15")
assert stu_map["sa:2"].departure == _dt("10:15")
assert "sa:3" in stu_map
assert stu_map["sa:3"].arrival == _dt("11:10")
assert stu_map["sa:3"].departure == _dt("11:10")
def test_delays_then_cancellation_in_2_updates(navitia_vj):
"""
Same test as above, but with nothing in the db, and with 2 updates
"""
with app.app_context():
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="update",
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.stop_time_updates = [
StopTimeUpdate({"id": "sa:1"},
departure_delay=timedelta(minutes=5),
dep_status="update")
]
handle(real_time_update, [trip_update],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="delete",
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
res, _ = handle(real_time_update, [trip_update],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert trip_update.status == "delete"
assert len(trip_update.stop_time_updates) == 0
assert len(trip_update.real_time_updates) == 2
def test_handle_basic():
with app.app_context():
contributor = model.Contributor(
id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value
)
builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor)
with pytest.raises(TypeError):
handle(builder, None, [])
# a RealTimeUpdate without any TripUpdate doesn't do anything
real_time_update = make_rt_update(raw_data=None, contributor_id=GTFS_CONTRIBUTOR_ID)
res, _ = handle(builder, real_time_update, [])
assert res == real_time_update
def test_handle_basic():
with pytest.raises(TypeError):
handle(None, [],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
# a RealTimeUpdate without any TripUpdate doesn't do anything
with app.app_context():
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
res, _ = handle(real_time_update, [],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
assert res == real_time_update
def handle(proto, navitia_wrapper, contributor):
data = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor)
start_datetime = datetime.datetime.utcnow()
try:
trip_updates = KirinModelBuilder(navitia_wrapper,
contributor).build(rt_update,
data=proto)
record_call('OK', contributor=contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=contributor)
raise
real_time_update, log_dict = core.handle(rt_update, trip_updates,
contributor)
# After merging trip_updates information of gtfs-rt, navitia and kirin database, if there is no new information
# destinated to navitia, update real_time_update with status = 'KO' and a proper error message.
if not real_time_update.trip_updates and real_time_update.status == 'OK':
real_time_update.status = 'KO'
real_time_update.error = 'No new information destinated to navitia for this gtfs-rt ' \
'with timestamp: {}'.format(proto.header.timestamp)
logging.getLogger(__name__).error(
'No new information destinated to navitia for this gtfs-rt '
'with timestamp: {}'.format(proto.header.timestamp))
model.db.session.add(rt_update)
model.db.session.commit()
duration = (datetime.datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({
'duration':
duration,
'input_timestamp':
datetime.datetime.utcfromtimestamp(proto.header.timestamp)
})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def build_rt_update(self, input_raw):
log_dict = {}
try:
ElementTree.fromstring(input_raw)
except ParseError:
# We save the non-decodable xml feed
rt_update = manage_db_error(
str(input_raw, encoding="utf-8",
errors="replace"), # protect for PostgreSQL "Text" type
contributor_id=self.contributor.id,
error="invalid xml",
is_reprocess_same_data_allowed=False,
)
return rt_update, log_dict
rt_update = make_rt_update(input_raw,
contributor_id=self.contributor.id)
return rt_update, log_dict
def handle(proto, navitia_wrapper, contributor):
data = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(data, 'gtfs-rt')
try:
trip_updates = KirinModelBuilder(navitia_wrapper, contributor).build(rt_update, data=proto)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
raise
core.handle(rt_update, trip_updates, contributor)
def test_handle_new_trip_out_of_order(navitia_vj):
"""
We have one vj with only one stop time updated, but it's not the first
so we have to reorder the stop times in the resulting trip_update
"""
with app.app_context():
trip_update = TripUpdate(_create_db_vj(navitia_vj),
status="update",
contributor_id=COTS_CONTRIBUTOR_ID)
st = StopTimeUpdate(
{"id": "sa:2"},
departure_delay=timedelta(minutes=40),
dep_status="update",
arrival_delay=timedelta(minutes=44),
arr_status="update",
order=1,
)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.stop_time_updates.append(st)
res, _ = handle(real_time_update, [trip_update],
contributor_id=COTS_CONTRIBUTOR_ID,
is_new_complete=False)
assert len(res.trip_updates) == 1
trip_update = res.trip_updates[0]
assert len(trip_update.stop_time_updates) == 3
assert trip_update.stop_time_updates[0].stop_id == "sa:1"
assert trip_update.stop_time_updates[0].departure == _dt("8:10")
assert trip_update.stop_time_updates[0].arrival == _dt("8:10")
assert trip_update.stop_time_updates[1].stop_id == "sa:2"
assert trip_update.stop_time_updates[1].departure == _dt("9:50")
assert trip_update.stop_time_updates[1].arrival == _dt("9:49")
assert trip_update.stop_time_updates[2].stop_id == "sa:3"
assert trip_update.stop_time_updates[2].departure == _dt("10:05")
assert trip_update.stop_time_updates[2].arrival == _dt("10:05")
def process_post(self, input_raw, contributor_type, is_new_complete=False):
# create a raw rt_update obj, save the raw_input into the db
rt_update = make_rt_update(input_raw,
contributor_type,
contributor=self.contributor)
start_datetime = datetime.utcnow()
try:
# assuming UTF-8 encoding for all input
rt_update.raw_data = rt_update.raw_data.encode('utf-8')
# raw_input is interpreted
trip_updates = self.builder(self.navitia_wrapper,
self.contributor).build(rt_update)
record_call('OK', contributor=self.contributor)
except KirinException as e:
rt_update.status = 'KO'
rt_update.error = e.data['error']
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
except Exception as e:
rt_update.status = 'KO'
rt_update.error = e.message
model.db.session.add(rt_update)
model.db.session.commit()
record_call('failure', reason=str(e), contributor=self.contributor)
raise
_, log_dict = core.handle(rt_update,
trip_updates,
self.contributor,
is_new_complete=is_new_complete)
duration = (datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({'duration': duration})
record_call('Simple feed publication', **log_dict)
logging.getLogger(__name__).info('Simple feed publication',
extra=log_dict)
return 'OK', 200
def build_rt_update(self, input_raw):
# create a raw gtfs-rt obj, save the raw protobuf into the db (as text)
proto = gtfs_realtime_pb2.FeedMessage()
log_dict = {}
try:
proto.ParseFromString(input_raw)
except DecodeError:
# We save the non-decodable gtfs-rt feed
rt_update = manage_db_error(
str(input_raw, encoding="utf-8", errors="replace"), # protect for PostgreSQL "Text" type
contributor_id=self.contributor.id,
error="invalid protobuf",
is_reprocess_same_data_allowed=False,
)
return rt_update, log_dict
feed = str(proto) # temp, for the moment, we save the protobuf as text
rt_update = make_rt_update(feed, contributor_id=self.contributor.id)
rt_update.proto = proto
return rt_update, log_dict
