def test_multiple_delays_in_2_updates(navitia_vj): """ same test as test_multiple_delays, but with nothing in the db and with 2 trip updates """ with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update") ] handle(builder, real_time_update, [trip_update]) trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=10), dep_status="update"), StopTimeUpdate({"id": "sa:2"}, arrival_delay=timedelta(minutes=2), arr_status="update"), ] res, _ = handle(builder, real_time_update, [trip_update]) _check_multiples_delay(res) # we also check that there is what we want in the db db_trip_updates = res.query.from_self(TripUpdate).all() assert len(db_trip_updates) == 1 assert db_trip_updates[0].status == "update" assert len(RealTimeUpdate.query.all()) == 2 assert len(StopTimeUpdate.query.all()) == 3
def test_cancellation_then_delay_in_2_updates(navitia_vj): """ same as test_cancellation_then_delay, but with a clear db and in 2 updates """ with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) trip_update = TripUpdate( _create_db_vj(navitia_vj), status="delete", effect=TripEffect.NO_SERVICE.name, contributor_id=contributor.id, ) trip_update.stop_time_updates = [] real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) handle(builder, real_time_update, [trip_update]) trip_update = TripUpdate( _create_db_vj(navitia_vj), status="none", effect=TripEffect.UNKNOWN_EFFECT.name, contributor_id=contributor.id, ) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:1"}, arr_status="none", dep_status="none", order=0), StopTimeUpdate({"id": "sa:2"}, arr_status="none", dep_status="none", order=1), StopTimeUpdate({"id": "sa:3"}, arrival_delay=timedelta(minutes=40), arr_status="update", order=2), ] res, _ = handle(builder, real_time_update, [trip_update]) _check_cancellation_then_delay(res)
def test_delays_then_cancellation_in_2_updates(navitia_vj): """ Same test as above, but with nothing in the db, and with 2 updates """ with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update") ] handle(builder, real_time_update, [trip_update]) trip_update = TripUpdate( _create_db_vj(navitia_vj), status="delete", effect=TripEffect.NO_SERVICE.name, contributor_id=contributor.id, ) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) res, _ = handle(builder, real_time_update, [trip_update]) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert trip_update.status == "delete" assert len(trip_update.stop_time_updates) == 3 for stu in trip_update.stop_time_updates: assert stu.arrival_status == "delete" assert stu.departure_status == "delete" assert len(trip_update.real_time_updates) == 2
def test_cancellation_then_delay_in_2_updates(navitia_vj): """ same as test_cancellation_then_delay, but with a clear db and in 2 updates """ with app.app_context(): trip_update = TripUpdate(_create_db_vj(navitia_vj), status="delete", contributor_id=COTS_CONTRIBUTOR_ID) trip_update.stop_time_updates = [] real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) handle(real_time_update, [trip_update], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=COTS_CONTRIBUTOR_ID) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:3"}, arrival_delay=timedelta(minutes=40), arr_status="update", order=2) ] res, _ = handle(real_time_update, [trip_update], COTS_CONTRIBUTOR_ID, is_new_complete=False) _check_cancellation_then_delay(res)
def test_past_midnight(): """ integration of a past midnight """ navitia_vj = { "trip": {"id": "vehicle_journey:1"}, "stop_times": [ { "utc_arrival_time": datetime.time(22, 10), "utc_departure_time": datetime.time(22, 15), "stop_point": {"id": "sa:1"}, }, # arrive at sa:2 at 23:10 and leave the day after { "utc_arrival_time": datetime.time(23, 10), "utc_departure_time": datetime.time(2, 15), "stop_point": {"id": "sa:2", "stop_area": {"timezone": "UTC"}}, }, { "utc_arrival_time": datetime.time(3, 20), "utc_departure_time": datetime.time(3, 25), "stop_point": {"id": "sa:3", "stop_area": {"timezone": "UTC"}}, }, ], } with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) vj = VehicleJourney( navitia_vj, datetime.datetime(2015, 9, 8, 21, 15, 0), datetime.datetime(2015, 9, 9, 4, 20, 0) ) trip_update = TripUpdate(vj, status="update", contributor_id=contributor.id) st = StopTimeUpdate({"id": "sa:2"}, departure_delay=timedelta(minutes=31), dep_status="update", order=1) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) trip_update.stop_time_updates.append(st) res, _ = handle(builder, real_time_update, [trip_update]) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert trip_update.status == "update" assert len(trip_update.stop_time_updates) == 3 assert trip_update.stop_time_updates[0].stop_id == "sa:1" assert trip_update.stop_time_updates[0].arrival == _dt("22:10") assert trip_update.stop_time_updates[0].departure == _dt("22:15") assert trip_update.stop_time_updates[1].stop_id == "sa:2" assert trip_update.stop_time_updates[1].arrival == _dt("23:10") assert trip_update.stop_time_updates[1].arrival_delay == timedelta(0) assert trip_update.stop_time_updates[1].departure == _dt("2:46", day=9) assert trip_update.stop_time_updates[1].departure_delay == timedelta(minutes=31) assert trip_update.stop_time_updates[2].stop_id == "sa:3" assert trip_update.stop_time_updates[2].arrival == _dt("3:51", day=9) assert trip_update.stop_time_updates[2].arrival_delay == timedelta(minutes=31) assert trip_update.stop_time_updates[2].departure == _dt("3:56", day=9) assert trip_update.stop_time_updates[2].departure_delay == timedelta(minutes=31)
def build_rt_update(self, input_raw): # create a raw gtfs-rt obj, save the raw protobuf into the db proto = gtfs_realtime_pb2.FeedMessage() log_dict = {} try: proto.ParseFromString(input_raw) except DecodeError: # We save the non-decodable flux gtfs-rt rt_update = manage_db_error( input_raw.encode( "string_escape", "ignore"), # protect for PostgreSQL "Text" type ConnectorType.gtfs_rt.value, contributor_id=self.contributor.id, error="invalid protobuf", is_reprocess_same_data_allowed=False, ) return rt_update, log_dict feed = six.binary_type( proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update( feed, connector_type=self.contributor.connector_type, contributor_id=self.contributor.id) rt_update.proto = proto return rt_update, log_dict
def handle(proto, navitia_wrapper, contributor): data = str(proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor) start_datetime = datetime.datetime.utcnow() try: trip_updates = KirinModelBuilder(navitia_wrapper, contributor).build(rt_update, data=proto) record_call('OK', contributor=contributor) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=contributor) raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=contributor) raise real_time_update, log_dict = core.handle(rt_update, trip_updates, contributor) duration = (datetime.datetime.utcnow() - start_datetime).total_seconds() log_dict.update({ 'duration': duration, 'input_timestamp': datetime.datetime.utcfromtimestamp(proto.header.timestamp) }) record_call('Simple feed publication', **log_dict) logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def create_real_time_update(id, contributor_id, connector_type, vj_id, trip_id, circulation_date): rtu = make_rt_update("", connector_type, contributor_id=contributor_id) rtu.id = id trip_update = create_trip_update(vj_id, trip_id, circulation_date) trip_update.contributor_id = contributor_id rtu.trip_updates.append(trip_update)
def test_delays_then_cancellation(setup_database, navitia_vj): """ We have a delay on the first st of a vj in the db and we receive a cancellation on this vj, we should have a cancelled vj in the end sa:1 sa:2 sa:3 VJ navitia 8:10 9:05-9:10 10:05 VJ in db 8:15* 9:05-9:10 10:05 update kirin - """ with app.app_context(): trip_update = TripUpdate(_create_db_vj(navitia_vj), status="delete", contributor_id=COTS_CONTRIBUTOR_ID) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) res, _ = handle(real_time_update, [trip_update], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert trip_update.status == "delete" assert len(trip_update.stop_time_updates) == 0 assert len(trip_update.real_time_updates) == 2
def post(self): raw_xml = get_ire(flask.globals.request) # create a raw ire obj, save the raw_xml into the db rt_update = make_rt_update(raw_xml, 'ire') try: # assuming UTF-8 encoding for all ire input rt_update.raw_data = rt_update.raw_data.encode('utf-8') # raw_xml is interpreted trip_updates = KirinModelBuilder(self.navitia_wrapper, self.contributor).build(rt_update) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() raise core.handle(rt_update, trip_updates, current_app.config['CONTRIBUTOR']) return 'OK', 200
def build_rt_update(self, input_raw): rt_update = make_rt_update( input_raw, connector_type=self.contributor.connector_type, contributor_id=self.contributor.id) log_dict = {} return rt_update, log_dict
def test_delays_then_cancellation(setup_database, navitia_vj): """ We have a delay on the first st of a vj in the db and we receive a cancellation on this vj, we should have a cancelled vj in the end sa:1 sa:2 sa:3 VJ navitia 8:10 9:05-9:10 10:05 VJ in db 8:15* 9:05-9:10 10:05 update kirin - """ with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) trip_update = TripUpdate( _create_db_vj(navitia_vj), status="delete", effect=TripEffect.NO_SERVICE.name, contributor_id=contributor.id, ) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) res, _ = handle(builder, real_time_update, [trip_update]) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert trip_update.status == "delete" assert len(trip_update.stop_time_updates) == 3 for stu in trip_update.stop_time_updates: assert stu.arrival_status == "delete" assert stu.departure_status == "delete" assert len(trip_update.real_time_updates) == 2
def test_multiple_delays(setup_database, navitia_vj): """ We receive a delay on the first and second stoptimes of a vj, and there was already some delay on the first st of this vj sa:1 sa:2 sa:3 VJ navitia 8:10 9:05-9:10 10:05 VJ in db 8:15* 9:05-9:10 10:05 update kirin 8:20* *9:07-9:10 10:05 """ with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) trip_update.stop_time_updates = [ # Note: the delay is based of the navitia's vj StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=10), dep_status="update"), StopTimeUpdate({"id": "sa:2"}, arrival_delay=timedelta(minutes=2), arr_status="update"), ] res, _ = handle(builder, real_time_update, [trip_update]) _check_multiples_delay(res) # we also check that there is what we want in the db db_trip_updates = res.query.from_self(TripUpdate).all() assert len(db_trip_updates) == 2 for tu in db_trip_updates: assert tu.status == "update" assert len(RealTimeUpdate.query.all()) == 3 # 2 already in db, one new update assert len(StopTimeUpdate.query.all()) == 6 # 3 st * 2 vj in the db
def test_multiple_delays_in_2_updates(navitia_vj): """ same test as test_multiple_delays, but with nothing in the db and with 2 trip updates """ with app.app_context(): trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=COTS_CONTRIBUTOR_ID) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update") ] handle(real_time_update, [trip_update], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=COTS_CONTRIBUTOR_ID) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=10), dep_status="update"), StopTimeUpdate({"id": "sa:2"}, arrival_delay=timedelta(minutes=2), arr_status="update"), ] res, _ = handle(real_time_update, [trip_update], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) _check_multiples_delay(res) # we also check that there is what we want in the db db_trip_updates = res.query.from_self(TripUpdate).all() assert len(db_trip_updates) == 1 assert db_trip_updates[0].status == "update" assert len(RealTimeUpdate.query.all()) == 2 assert len(StopTimeUpdate.query.all()) == 3
def test_handle_new_vj(): """an easy one: we have one vj with only one stop time updated""" navitia_vj = { "trip": {"id": "vehicle_journey:1"}, "stop_times": [ { "utc_arrival_time": None, "utc_departure_time": datetime.time(8, 10), "stop_point": {"id": "sa:1", "stop_area": {"timezone": "UTC"}}, }, { "utc_arrival_time": datetime.time(9, 10), "utc_departure_time": None, "stop_point": {"id": "sa:2", "stop_area": {"timezone": "UTC"}}, }, ], } with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) trip_update = TripUpdate(_create_db_vj(navitia_vj), contributor_id=contributor.id, status="update") st = StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update") real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) trip_update.stop_time_updates.append(st) res, _ = handle(builder, real_time_update, [trip_update]) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert trip_update.status == "update" assert len(trip_update.stop_time_updates) == 2 assert trip_update.stop_time_updates[0].stop_id == "sa:1" assert trip_update.stop_time_updates[0].departure == _dt("8:15") assert trip_update.stop_time_updates[0].arrival == _dt("8:15") assert trip_update.stop_time_updates[1].stop_id == "sa:2" assert trip_update.stop_time_updates[1].departure == _dt("9:15") assert trip_update.stop_time_updates[1].arrival == _dt("9:15") # testing that RealTimeUpdate is persisted in db db_trip_updates = real_time_update.query.from_self(TripUpdate).all() assert len(db_trip_updates) == 1 assert db_trip_updates[0].status == "update" db_st_updates = real_time_update.query.from_self(StopTimeUpdate).order_by("stop_id").all() assert len(db_st_updates) == 2 assert db_st_updates[0].stop_id == "sa:1" assert db_st_updates[0].departure == _dt("8:15") assert db_st_updates[0].arrival == _dt("8:15") assert db_st_updates[0].trip_update_id == db_trip_updates[0].vj_id assert db_st_updates[1].stop_id == "sa:2" assert db_st_updates[1].departure == _dt("9:15") assert db_st_updates[1].arrival == _dt("9:15") assert db_st_updates[1].trip_update_id == db_trip_updates[0].vj_id
def test_cancellation_then_delay(navitia_vj): """ we have a cancelled vj in the db, and we receive an update on the 3rd stoptime, at the end we have a delayed vj sa:1 sa:2 sa:3 VJ navitia 8:10 9:05-9:10 10:05 VJ in db - update kirin 8:10 9:05-9:10 10:45* """ with app.app_context(): vju = create_trip_update( "70866ce8-0638-4fa1-8556-1ddfa22d09d3", "vehicle_journey:1", datetime.date(2015, 9, 8), [], status="delete", ) rtu = make_rt_update(None, ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) rtu.id = "10866ce8-0638-4fa1-8556-1ddfa22d09d3" rtu.trip_updates.append(vju) db.session.add(rtu) db.session.commit() with app.app_context(): trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=COTS_CONTRIBUTOR_ID) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:3"}, arrival_delay=timedelta(minutes=40), arr_status="update", order=2) ] res, _ = handle(real_time_update, [trip_update], COTS_CONTRIBUTOR_ID, is_new_complete=False) _check_cancellation_then_delay(res)
def test_populate_pb_with_cancelation(): """ VJ cancelation """ navitia_vj = { "trip": { "id": "vehicle_journey:1" }, "stop_times": [{ "utc_arrival_time": datetime.time(8, 10), "stop_point": { "stop_area": { "timezone": "UTC" } } }], } with app.app_context(): vj = VehicleJourney(navitia_vj, datetime.datetime(2015, 9, 8, 7, 10, 0), datetime.datetime(2015, 9, 8, 11, 5, 0)) trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.vj = vj trip_update.status = "delete" trip_update.message = "Message Test" real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.company_id = "sncf" trip_update.effect = "REDUCED_SERVICE" real_time_update.trip_updates.append(trip_update) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version == "1" pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == "vehicle_journey:1" assert pb_trip_update.trip.start_date == "20150908" assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is True assert pb_trip_update.Extensions[ kirin_pb2.trip_message] == "Message Test" assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True assert pb_trip_update.trip.Extensions[ kirin_pb2.contributor] == COTS_CONTRIBUTOR_ID assert pb_trip_update.trip.Extensions[kirin_pb2.company_id] == "sncf" assert pb_trip_update.Extensions[ kirin_pb2.effect] == gtfs_realtime_pb2.Alert.REDUCED_SERVICE assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_simple_delay(navitia_vj): """Test on delay when there is nothing in the db""" with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id) st = StopTimeUpdate( {"id": "sa:1"}, arrival_delay=timedelta(minutes=5), arr_status="update", departure_delay=timedelta(minutes=10), dep_status="update", order=0, ) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) trip_update.stop_time_updates.append(st) res, _ = handle(builder, real_time_update, [trip_update]) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert trip_update.status == "update" assert len(trip_update.stop_time_updates) == 3 assert trip_update.stop_time_updates[0].stop_id == "sa:1" assert trip_update.stop_time_updates[0].departure == _dt("8:20") # 8:10 + 10mn assert trip_update.stop_time_updates[0].departure_delay == timedelta(minutes=10) assert trip_update.stop_time_updates[0].departure_status == "update" assert trip_update.stop_time_updates[1].stop_id == "sa:2" assert trip_update.stop_time_updates[1].arrival == _dt("9:15") assert trip_update.stop_time_updates[1].arrival_delay == timedelta(minutes=10) assert trip_update.stop_time_updates[1].arrival_status == "update" assert trip_update.stop_time_updates[1].departure == _dt("9:20") assert trip_update.stop_time_updates[1].departure_delay == timedelta(minutes=10) assert trip_update.stop_time_updates[1].departure_status == "update" # testing that RealTimeUpdate is persisted in db db_trip_updates = real_time_update.query.from_self(TripUpdate).all() assert len(db_trip_updates) == 1 assert db_trip_updates[0].status == "update" db_st_updates = real_time_update.query.from_self(StopTimeUpdate).order_by("stop_id").all() assert len(db_st_updates) == 3 assert db_st_updates[0].stop_id == "sa:1" assert db_st_updates[0].departure == _dt("8:20") # 8:10 + 10mn assert db_st_updates[0].departure_delay == timedelta(minutes=10) assert db_st_updates[0].departure_status == "update" assert db_st_updates[1].stop_id == "sa:2" assert db_st_updates[1].arrival == _dt("9:15") assert db_st_updates[1].departure == _dt("9:20") assert db_st_updates[1].trip_update_id == db_trip_updates[0].vj_id assert db_st_updates[2].stop_id == "sa:3"
def test_cancellation_then_delay(navitia_vj): """ we have a cancelled vj in the db, and we receive an update on the 3rd stoptime, at the end we have a delayed vj sa:1 sa:2 sa:3 VJ navitia 8:10 9:05-9:10 10:05 VJ in db - update kirin 8:10 9:05-9:10 10:45* """ with app.app_context(): vju = create_trip_update( "70866ce8-0638-4fa1-8556-1ddfa22d09d3", "vehicle_journey:1", datetime.date(2015, 9, 8), [], status="delete", ) rtu = make_rt_update(None, contributor_id=GTFS_CONTRIBUTOR_ID) rtu.id = "10866ce8-0638-4fa1-8556-1ddfa22d09d3" rtu.trip_updates.append(vju) db_commit(rtu) with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) trip_update = TripUpdate( _create_db_vj(navitia_vj), status="update", effect=TripEffect.UNKNOWN_EFFECT.name, contributor_id=contributor.id, ) real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:3"}, arrival_delay=timedelta(minutes=40), arr_status="update", order=2) ] res, _ = handle(builder, real_time_update, [trip_update]) _check_cancellation_then_delay(res)
def test_manage_consistency(navitia_vj): """ we receive an update for a vj already in the database sa:1 sa:2 sa:3 VJ navitia 08:10 09:05-09:10 10:05 update kirin - *10:15-09:20* - expected result 08:10-08:10 10:15-10:15 10:15-10:15 > propagating 10 min from sa:2 departure to sa:3, not needing correction for consistency """ with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=contributor.id) st = StopTimeUpdate( {"id": "sa:2"}, arrival_delay=timedelta(minutes=70), dep_status="update", departure_delay=timedelta(minutes=10), arr_status="update", order=1, ) st.arrival_status = st.departure_status = "update" real_time_update = make_rt_update(raw_data=None, contributor_id=contributor.id) real_time_update.id = "30866ce8-0638-4fa1-8556-1ddfa22d09d3" trip_update.stop_time_updates.append(st) res, _ = handle(builder, real_time_update, [trip_update]) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert trip_update.status == "update" assert len(trip_update.real_time_updates) == 1 assert len(trip_update.stop_time_updates) == 3 stu_map = {stu.stop_id: stu for stu in trip_update.stop_time_updates} assert "sa:1" in stu_map assert stu_map["sa:1"].arrival == _dt("8:10") assert stu_map["sa:1"].departure == _dt("8:10") assert "sa:2" in stu_map assert stu_map["sa:2"].arrival == _dt("10:15") assert stu_map["sa:2"].departure == _dt("10:15") assert "sa:3" in stu_map assert stu_map["sa:3"].arrival == _dt("10:15") assert stu_map["sa:3"].departure == _dt("10:15")
def test_manage_consistency(navitia_vj): """ we receive an update for a vj already in the database sa:1 sa:2 sa:3 VJ navitia 08:10 09:05-09:10 10:05 update kirin - *10:15-09:20* - expected result 08:10-08:10 10:15-10:15 11:10-11:10 """ with app.app_context(): trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=COTS_CONTRIBUTOR_ID) st = StopTimeUpdate( {"id": "sa:2"}, arrival_delay=timedelta(minutes=70), dep_status="update", departure_delay=timedelta(minutes=10), arr_status="update", order=1, ) st.arrival_status = st.departure_status = "update" real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) real_time_update.id = "30866ce8-0638-4fa1-8556-1ddfa22d09d3" trip_update.stop_time_updates.append(st) res, _ = handle(real_time_update, [trip_update], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert trip_update.status == "update" assert len(trip_update.real_time_updates) == 1 assert len(trip_update.stop_time_updates) == 3 stu_map = {stu.stop_id: stu for stu in trip_update.stop_time_updates} assert "sa:1" in stu_map assert stu_map["sa:1"].arrival == _dt("8:10") assert stu_map["sa:1"].departure == _dt("8:10") assert "sa:2" in stu_map assert stu_map["sa:2"].arrival == _dt("10:15") assert stu_map["sa:2"].departure == _dt("10:15") assert "sa:3" in stu_map assert stu_map["sa:3"].arrival == _dt("11:10") assert stu_map["sa:3"].departure == _dt("11:10")
def test_delays_then_cancellation_in_2_updates(navitia_vj): """ Same test as above, but with nothing in the db, and with 2 updates """ with app.app_context(): trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=COTS_CONTRIBUTOR_ID) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.stop_time_updates = [ StopTimeUpdate({"id": "sa:1"}, departure_delay=timedelta(minutes=5), dep_status="update") ] handle(real_time_update, [trip_update], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) trip_update = TripUpdate(_create_db_vj(navitia_vj), status="delete", contributor_id=COTS_CONTRIBUTOR_ID) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) res, _ = handle(real_time_update, [trip_update], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert trip_update.status == "delete" assert len(trip_update.stop_time_updates) == 0 assert len(trip_update.real_time_updates) == 2
def test_handle_basic(): with app.app_context(): contributor = model.Contributor( id=GTFS_CONTRIBUTOR_ID, navitia_coverage=None, connector_type=ConnectorType.gtfs_rt.value ) builder = kirin.poller_workers.gtfs_rt.KirinModelBuilder(contributor) with pytest.raises(TypeError): handle(builder, None, []) # a RealTimeUpdate without any TripUpdate doesn't do anything real_time_update = make_rt_update(raw_data=None, contributor_id=GTFS_CONTRIBUTOR_ID) res, _ = handle(builder, real_time_update, []) assert res == real_time_update
def test_handle_basic(): with pytest.raises(TypeError): handle(None, [], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) # a RealTimeUpdate without any TripUpdate doesn't do anything with app.app_context(): real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) res, _ = handle(real_time_update, [], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) assert res == real_time_update
def handle(proto, navitia_wrapper, contributor): data = str(proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update(data, 'gtfs-rt', contributor=contributor) start_datetime = datetime.datetime.utcnow() try: trip_updates = KirinModelBuilder(navitia_wrapper, contributor).build(rt_update, data=proto) record_call('OK', contributor=contributor) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=contributor) raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=contributor) raise real_time_update, log_dict = core.handle(rt_update, trip_updates, contributor) # After merging trip_updates information of gtfs-rt, navitia and kirin database, if there is no new information # destinated to navitia, update real_time_update with status = 'KO' and a proper error message. if not real_time_update.trip_updates and real_time_update.status == 'OK': real_time_update.status = 'KO' real_time_update.error = 'No new information destinated to navitia for this gtfs-rt ' \ 'with timestamp: {}'.format(proto.header.timestamp) logging.getLogger(__name__).error( 'No new information destinated to navitia for this gtfs-rt ' 'with timestamp: {}'.format(proto.header.timestamp)) model.db.session.add(rt_update) model.db.session.commit() duration = (datetime.datetime.utcnow() - start_datetime).total_seconds() log_dict.update({ 'duration': duration, 'input_timestamp': datetime.datetime.utcfromtimestamp(proto.header.timestamp) }) record_call('Simple feed publication', **log_dict) logging.getLogger(__name__).info('Simple feed publication', extra=log_dict)
def build_rt_update(self, input_raw): log_dict = {} try: ElementTree.fromstring(input_raw) except ParseError: # We save the non-decodable xml feed rt_update = manage_db_error( str(input_raw, encoding="utf-8", errors="replace"), # protect for PostgreSQL "Text" type contributor_id=self.contributor.id, error="invalid xml", is_reprocess_same_data_allowed=False, ) return rt_update, log_dict rt_update = make_rt_update(input_raw, contributor_id=self.contributor.id) return rt_update, log_dict
def handle(proto, navitia_wrapper, contributor): data = str(proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update(data, 'gtfs-rt') try: trip_updates = KirinModelBuilder(navitia_wrapper, contributor).build(rt_update, data=proto) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() raise core.handle(rt_update, trip_updates, contributor)
def test_handle_new_trip_out_of_order(navitia_vj): """ We have one vj with only one stop time updated, but it's not the first so we have to reorder the stop times in the resulting trip_update """ with app.app_context(): trip_update = TripUpdate(_create_db_vj(navitia_vj), status="update", contributor_id=COTS_CONTRIBUTOR_ID) st = StopTimeUpdate( {"id": "sa:2"}, departure_delay=timedelta(minutes=40), dep_status="update", arrival_delay=timedelta(minutes=44), arr_status="update", order=1, ) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.stop_time_updates.append(st) res, _ = handle(real_time_update, [trip_update], contributor_id=COTS_CONTRIBUTOR_ID, is_new_complete=False) assert len(res.trip_updates) == 1 trip_update = res.trip_updates[0] assert len(trip_update.stop_time_updates) == 3 assert trip_update.stop_time_updates[0].stop_id == "sa:1" assert trip_update.stop_time_updates[0].departure == _dt("8:10") assert trip_update.stop_time_updates[0].arrival == _dt("8:10") assert trip_update.stop_time_updates[1].stop_id == "sa:2" assert trip_update.stop_time_updates[1].departure == _dt("9:50") assert trip_update.stop_time_updates[1].arrival == _dt("9:49") assert trip_update.stop_time_updates[2].stop_id == "sa:3" assert trip_update.stop_time_updates[2].departure == _dt("10:05") assert trip_update.stop_time_updates[2].arrival == _dt("10:05")
def process_post(self, input_raw, contributor_type, is_new_complete=False): # create a raw rt_update obj, save the raw_input into the db rt_update = make_rt_update(input_raw, contributor_type, contributor=self.contributor) start_datetime = datetime.utcnow() try: # assuming UTF-8 encoding for all input rt_update.raw_data = rt_update.raw_data.encode('utf-8') # raw_input is interpreted trip_updates = self.builder(self.navitia_wrapper, self.contributor).build(rt_update) record_call('OK', contributor=self.contributor) except KirinException as e: rt_update.status = 'KO' rt_update.error = e.data['error'] model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=self.contributor) raise except Exception as e: rt_update.status = 'KO' rt_update.error = e.message model.db.session.add(rt_update) model.db.session.commit() record_call('failure', reason=str(e), contributor=self.contributor) raise _, log_dict = core.handle(rt_update, trip_updates, self.contributor, is_new_complete=is_new_complete) duration = (datetime.utcnow() - start_datetime).total_seconds() log_dict.update({'duration': duration}) record_call('Simple feed publication', **log_dict) logging.getLogger(__name__).info('Simple feed publication', extra=log_dict) return 'OK', 200
def build_rt_update(self, input_raw): # create a raw gtfs-rt obj, save the raw protobuf into the db (as text) proto = gtfs_realtime_pb2.FeedMessage() log_dict = {} try: proto.ParseFromString(input_raw) except DecodeError: # We save the non-decodable gtfs-rt feed rt_update = manage_db_error( str(input_raw, encoding="utf-8", errors="replace"), # protect for PostgreSQL "Text" type contributor_id=self.contributor.id, error="invalid protobuf", is_reprocess_same_data_allowed=False, ) return rt_update, log_dict feed = str(proto) # temp, for the moment, we save the protobuf as text rt_update = make_rt_update(feed, contributor_id=self.contributor.id) rt_update.proto = proto return rt_update, log_dict