def handle(real_time_update, trip_updates): """ receive a RealTimeUpdate with at least one TripUpdate filled with the data received by the connector. each TripUpdate is associated with the VehicleJourney returned by jormugandr """ if not real_time_update: raise TypeError() for trip_update in trip_updates: #find if there already a row in db old = TripUpdate.find_by_dated_vj(trip_update.vj.navitia_id, trip_update.vj.circulation_date) #merge the theoric, the current realtime, and the new relatime current_trip_update = merge(trip_update, old) # we have to link the current_vj_update with the new real_time_update # this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy current_trip_update.real_time_updates.append(real_time_update) persist(real_time_update) feed = convert_to_gtfsrt([real_time_update]) publish(feed, real_time_update) return real_time_update
def handle(real_time_update, trip_updates, contributor): """ receive a RealTimeUpdate with at least one TripUpdate filled with the data received by the connector. each TripUpdate is associated with the VehicleJourney returned by jormugandr """ if not real_time_update: raise TypeError() for trip_update in trip_updates: # find if there already a row in db old = TripUpdate.find_by_dated_vj(trip_update.vj.navitia_trip_id, trip_update.vj.circulation_date) # merge the theoric, the current realtime, and the new realtime current_trip_update = merge(trip_update.vj.navitia_vj, old, trip_update) # manage and adjust consistency if possible if manage_consistency(current_trip_update): # we have to link the current_vj_update with the new real_time_update # this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy current_trip_update.real_time_updates.append(real_time_update) persist(real_time_update) feed = convert_to_gtfsrt(real_time_update.trip_updates) publish(feed, contributor) return real_time_update
def test_populate_pb_with_cancelation(): """ VJ cancelation """ navitia_vj = {'id': 'vehicle_journey:1'} with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8)) trip_update.vj = vj trip_update.status = 'delete' real_time_update = RealTimeUpdate(raw_data=None, connector='ire') real_time_update.trip_updates.append(trip_update) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version == '1' pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_full_dataset(): """ VJ cancelation """ navitia_vj = {'trip': {'id': 'vehicle_journey:1'}} with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8)) trip_update.vj = vj trip_update.status = 'delete' trip_update.message = 'Message Test' real_time_update = RealTimeUpdate(raw_data=None, connector='ire') trip_update.contributor = 'kisio-digital' real_time_update.trip_updates.append(trip_update) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates, gtfs_realtime_pb2.FeedHeader.FULL_DATASET) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.FULL_DATASET assert feed_entity.header.gtfs_realtime_version == '1' pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == True assert pb_trip_update.Extensions[kirin_pb2.trip_message] == 'Message Test' assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == True assert pb_trip_update.trip.Extensions[kirin_pb2.contributor] == 'kisio-digital' assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def handle(builder, real_time_update, trip_updates): """ Receive a RealTimeUpdate with at least one TripUpdate filled with the data received by the connector. Each TripUpdate is associated with the base-schedule VehicleJourney, complete/merge realtime is done using builder Then persist in db, publish for Navitia Returns real_time_update and the log_dict """ if not real_time_update: raise TypeError() id_timestamp_tuples = [(tu.vj.navitia_trip_id, tu.vj.start_timestamp) for tu in trip_updates] old_trip_updates = TripUpdate.find_by_dated_vjs(id_timestamp_tuples) for trip_update in trip_updates: # find if there is already a row in db old = next( (tu for tu in old_trip_updates if tu.vj.navitia_trip_id == trip_update.vj.navitia_trip_id and tu.vj.start_timestamp == trip_update.vj.start_timestamp), None, ) # merge the base schedule, the current realtime, and the new realtime current_trip_update = builder.merge_trip_updates( trip_update.vj.navitia_vj, old, trip_update) # manage and adjust consistency if possible if current_trip_update is not None and check_consistency( current_trip_update): # we have to link the current_vj_update with the new real_time_update # this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy current_trip_update.real_time_updates.append(real_time_update) db_commit(real_time_update) feed = convert_to_gtfsrt(real_time_update.trip_updates, gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL) feed_str = feed.SerializeToString() publish(feed_str, builder.contributor.id) data_time = datetime.datetime.utcfromtimestamp(feed.header.timestamp) log_dict = { "contributor": builder.contributor.id, "timestamp": data_time, "output_trip_update_count": len(feed.entity), "output_feed_size": sys.getsizeof(feed_str), } # After merging trip_updates information of connector realtime, navitia and kirin database, if there is no new # information destined to navitia, update real_time_update with status = 'KO' and a proper error message. if not real_time_update.trip_updates and real_time_update.status == "pending": msg = "No new information destined for navitia on {}".format( real_time_update.contributor_id) set_rtu_status_ko(real_time_update, msg, is_reprocess_same_data_allowed=False) logging.getLogger(__name__).warning("RealTimeUpdate id={}: {}".format( real_time_update.id, msg), extra=log_dict) db_commit(real_time_update) return real_time_update, log_dict
def handle(real_time_update, trip_updates, contributor, is_new_complete=False): """ receive a RealTimeUpdate with at least one TripUpdate filled with the data received by the connector. each TripUpdate is associated with the VehicleJourney returned by jormugandr Returns real_time_update and the log_dict """ if not real_time_update: raise TypeError() id_timestamp_tuples = [(tu.vj.navitia_trip_id, tu.vj.get_start_timestamp()) for tu in trip_updates] old_trip_updates = TripUpdate.find_by_dated_vjs(id_timestamp_tuples) for trip_update in trip_updates: # find if there is already a row in db old = next((tu for tu in old_trip_updates if tu.vj.navitia_trip_id == trip_update.vj.navitia_trip_id and tu.vj.get_start_timestamp() == trip_update.vj.get_start_timestamp()), None) # merge the base schedule, the current realtime, and the new realtime current_trip_update = merge(trip_update.vj.navitia_vj, old, trip_update, is_new_complete=is_new_complete) # manage and adjust consistency if possible if current_trip_update and manage_consistency(current_trip_update): # we have to link the current_vj_update with the new real_time_update # this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy current_trip_update.real_time_updates.append(real_time_update) persist(real_time_update) feed = convert_to_gtfsrt(real_time_update.trip_updates) feed_str = feed.SerializeToString() publish(feed_str, contributor) data_time = datetime.datetime.utcfromtimestamp(feed.header.timestamp) log_dict = {'contributor': contributor, 'timestamp': data_time, 'trip_update_count': len(feed.entity), 'size': len(feed_str)} return real_time_update, log_dict
def test_populate_pb_with_cancelation(): """ VJ cancelation """ navitia_vj = { 'trip': { 'id': 'vehicle_journey:1' }, 'stop_times': [{ 'utc_arrival_time': datetime.time(8, 10), 'stop_point': { 'stop_area': { 'timezone': 'UTC' } } }] } with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney( navitia_vj, utc.localize(datetime.datetime(2015, 9, 8, 7, 10, 0)), utc.localize(datetime.datetime(2015, 9, 8, 11, 5, 0))) trip_update.vj = vj trip_update.status = 'delete' trip_update.message = 'Message Test' real_time_update = RealTimeUpdate(raw_data=None, connector='cots', contributor='realtime.cots') trip_update.contributor = 'kisio-digital' trip_update.company_id = 'sncf' trip_update.effect = 'REDUCED_SERVICE' real_time_update.trip_updates.append(trip_update) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version == '1' pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == True assert pb_trip_update.Extensions[ kirin_pb2.trip_message] == 'Message Test' assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == True assert pb_trip_update.trip.Extensions[ kirin_pb2.contributor] == 'kisio-digital' assert pb_trip_update.trip.Extensions[kirin_pb2.company_id] == 'sncf' assert pb_trip_update.Extensions[ kirin_pb2.effect] == gtfs_realtime_pb2.Alert.REDUCED_SERVICE assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_cancelation(): """ VJ cancelation """ navitia_vj = { "trip": { "id": "vehicle_journey:1" }, "stop_times": [{ "utc_arrival_time": datetime.time(8, 10), "stop_point": { "stop_area": { "timezone": "UTC" } } }], } with app.app_context(): vj = VehicleJourney(navitia_vj, datetime.datetime(2015, 9, 8, 7, 10, 0), datetime.datetime(2015, 9, 8, 11, 5, 0)) trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.vj = vj trip_update.status = "delete" trip_update.message = "Message Test" real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.company_id = "sncf" trip_update.effect = "REDUCED_SERVICE" real_time_update.trip_updates.append(trip_update) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version == "1" pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == "vehicle_journey:1" assert pb_trip_update.trip.start_date == "20150908" assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is True assert pb_trip_update.Extensions[ kirin_pb2.trip_message] == "Message Test" assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True assert pb_trip_update.trip.Extensions[ kirin_pb2.contributor] == COTS_CONTRIBUTOR_ID assert pb_trip_update.trip.Extensions[kirin_pb2.company_id] == "sncf" assert pb_trip_update.Extensions[ kirin_pb2.effect] == gtfs_realtime_pb2.Alert.REDUCED_SERVICE assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_full_dataset(): """ VJ cancelation """ navitia_vj = { 'trip': { 'id': 'vehicle_journey:1' }, 'stop_times': [{ 'arrival_time': datetime.time(8, 10), 'stop_point': { 'stop_area': { 'timezone': 'UTC' } } }] } with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8)) trip_update.vj = vj trip_update.status = 'delete' trip_update.message = 'Message Test' real_time_update = RealTimeUpdate(raw_data=None, connector='ire', contributor='realtime.ire') trip_update.contributor = 'kisio-digital' trip_update.company_id = 'keolis' trip_update.effect = 'DETOUR' real_time_update.trip_updates.append(trip_update) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt( real_time_update.trip_updates, gtfs_realtime_pb2.FeedHeader.FULL_DATASET) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.FULL_DATASET assert feed_entity.header.gtfs_realtime_version == '1' pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == True assert pb_trip_update.Extensions[ kirin_pb2.trip_message] == 'Message Test' assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == True assert pb_trip_update.trip.Extensions[ kirin_pb2.contributor] == 'kisio-digital' assert pb_trip_update.trip.Extensions[kirin_pb2.company_id] == 'keolis' assert pb_trip_update.Extensions[ kirin_pb2.effect] == gtfs_realtime_pb2.Alert.DETOUR assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_two_stop_time(): """ an easy one: we have one vj with only one stop time updated fill protobuf from trip_update Verify protobuf """ #we add another impacted stop time to the Model navitia_vj = {'trip': {'id': 'vehicle_journey:1'}, 'stop_times': [ {'arrival_time': None, 'departure_time': datetime.time(8, 10), 'stop_point': {'id': 'sa:1'}}, {'arrival_time': datetime.time(9, 10), 'departure_time': None, 'stop_point': {'id': 'sa:2'}} ]} with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8)) trip_update.vj = vj st = StopTimeUpdate({'id': 'sa:1'}, departure=_dt("8:15"), arrival=None) real_time_update = RealTimeUpdate(raw_data=None, connector='ire') real_time_update.trip_updates.append(trip_update) trip_update.stop_time_updates.append(st) st = StopTimeUpdate({'id': 'sa:2'}, departure=_dt("8:21"), arrival=_dt("8:20"), message="bob's on the track") real_time_update.trip_updates[0].stop_time_updates.append(st) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version, '1' assert len(feed_entity.entity) == 1 pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == False assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == False assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED assert len(pb_trip_update.stop_time_update) == 2 pb_stop_time = pb_trip_update.stop_time_update[0] assert pb_stop_time.stop_id == 'sa:1' assert pb_stop_time.arrival.time == 0 assert pb_stop_time.departure.time == to_posix_time(_dt("8:15")) assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == "" pb_stop_time = pb_trip_update.stop_time_update[1] assert pb_stop_time.stop_id == 'sa:2' assert pb_stop_time.arrival.time == to_posix_time(_dt("8:20")) assert pb_stop_time.departure.time == to_posix_time(_dt("8:21")) assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == "bob's on the track"
def _on_request(self, message): log = logging.getLogger(__name__) try: task = task_pb2.Task() try: # `body` is of unicode type, but we need str type for # `ParseFromString()` to work. It seems to work. # Maybe kombu estimate that, without any information, # the body should be something as json, and thus a # unicode string. On the c++ side, I didn't manage to # find a way to give a content-type or something like # that. body = str(message.payload) task.ParseFromString(body) except DecodeError as e: log.warn('invalid protobuf: {}'.format(str(e))) return log.info('Getting a full feed publication request', extra={'task': task}) if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime: return start_datetime = datetime.utcnow() begin_date = None end_date = None if hasattr(task.load_realtime, "begin_date"): if task.load_realtime.begin_date: begin_date = str_to_date(task.load_realtime.begin_date) if hasattr(task.load_realtime, "end_date"): if task.load_realtime.end_date: end_date = str_to_date(task.load_realtime.end_date) feed = convert_to_gtfsrt(TripUpdate.find_by_contributor_period(task.load_realtime.contributors, begin_date, end_date), gtfs_realtime_pb2.FeedHeader.FULL_DATASET) feed_str = feed.SerializeToString() log.info('Starting of full feed publication {}, {}'.format(len(feed_str), task), extra={'size': len(feed_str), 'task': task}) # http://docs.celeryproject.org/projects/kombu/en/latest/userguide/producers.html#bypassing-routing-by-using-the-anon-exchange self.producer.publish(feed_str, routing_key=task.load_realtime.queue_name, retry=True, retry_policy={ 'interval_start': 0, # First retry immediately, 'interval_step': 2, # then increase by 2s for every retry. 'interval_max': 10, # but don't exceed 10s between retries. 'max_retries': self.max_retries, # give up after 10 (by default) tries. }) duration = (datetime.utcnow() - start_datetime).total_seconds() log.info('End of full feed publication', extra={'duration': duration, 'task': task}) record_call('Full feed publication', size=len(feed_str), routing_key=task.load_realtime.queue_name, duration=duration, trip_update_count=len(feed.entity), contributor=task.load_realtime.contributors) finally: db.session.remove()
def handle(real_time_update, trip_updates, contributor, is_new_complete=False): """ receive a RealTimeUpdate with at least one TripUpdate filled with the data received by the connector. each TripUpdate is associated with the VehicleJourney returned by jormungandr Returns real_time_update and the log_dict """ if not real_time_update: raise TypeError() id_timestamp_tuples = [(tu.vj.navitia_trip_id, tu.vj.get_start_timestamp()) for tu in trip_updates] old_trip_updates = TripUpdate.find_by_dated_vjs(id_timestamp_tuples) for trip_update in trip_updates: # find if there is already a row in db old = next( (tu for tu in old_trip_updates if tu.vj.navitia_trip_id == trip_update.vj.navitia_trip_id and tu. vj.get_start_timestamp() == trip_update.vj.get_start_timestamp()), None) # merge the base schedule, the current realtime, and the new realtime current_trip_update = merge(trip_update.vj.navitia_vj, old, trip_update, is_new_complete=is_new_complete) # manage and adjust consistency if possible if current_trip_update and manage_consistency(current_trip_update): # we have to link the current_vj_update with the new real_time_update # this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy current_trip_update.real_time_updates.append(real_time_update) persist(real_time_update) feed = convert_to_gtfsrt(real_time_update.trip_updates) feed_str = feed.SerializeToString() publish(feed_str, contributor) data_time = datetime.datetime.utcfromtimestamp(feed.header.timestamp) log_dict = { 'contributor': contributor, 'timestamp': data_time, 'trip_update_count': len(feed.entity), 'size': len(feed_str) } # After merging trip_updates information of connector realtime, navitia and kirin database, if there is no new # information destined to navitia, update real_time_update with status = 'KO' and a proper error message. if not real_time_update.trip_updates and real_time_update.status == 'OK': real_time_update.status = 'KO' real_time_update.error = 'No new information destinated to navitia for this {}'.format( real_time_update.connector) logging.getLogger(__name__).error('RealTimeUpdate id={}: {}'.format( real_time_update.id, real_time_update.error)) model.db.session.add(real_time_update) model.db.session.commit() return real_time_update, log_dict
def test_populate_pb_with_one_stop_time(): """ an easy one: we have one vj with only one stop time updated fill protobuf from trip_update Verify protobuf """ navitia_vj = { 'trip': { 'id': 'vehicle_journey:1' }, 'stop_times': [{ 'arrival_time': None, 'departure_time': datetime.time(8, 10), 'stop_point': { 'id': 'sa:1' } }, { 'arrival_time': datetime.time(9, 10), 'departure_time': None, 'stop_point': { 'id': 'sa:2' } }] } with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8)) trip_update.vj = vj st = StopTimeUpdate({'id': 'sa:1'}, departure=_dt("8:15"), arrival=None) real_time_update = RealTimeUpdate(raw_data=None, connector='ire') real_time_update.trip_updates.append(trip_update) trip_update.stop_time_updates.append(st) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version == '1' pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED pb_stop_time = feed_entity.entity[0].trip_update.stop_time_update[0] assert pb_stop_time.arrival.time == 0 assert pb_stop_time.departure.time == to_posix_time(_dt("8:15")) assert pb_stop_time.stop_id == 'sa:1' assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == False assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == False
def callback(body, message): try: task = task_pb2.Task() try: # `body` is of unicode type, but we need str type for # `ParseFromString()` to work. It seems to work. # Maybe kombu estimate that, without any information, # the body should be something as json, and thus a # unicode string. On the c++ side, I didn't manage to # find a way to give a content-type or something like # that. body = str(body) task.ParseFromString(body) except DecodeError as e: log.warn('invalid protobuf: {}'.format(str(e))) return log.info('getting a request: {}'.format(task)) if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime: return begin_date = None end_date = None if hasattr(task.load_realtime, "begin_date"): if task.load_realtime.begin_date: begin_date = str_to_date(task.load_realtime.begin_date) if hasattr(task.load_realtime, "end_date"): if task.load_realtime.end_date: end_date = str_to_date(task.load_realtime.end_date) feed = convert_to_gtfsrt(TripUpdate.find_by_contributor_period(task.load_realtime.contributors, begin_date, end_date), gtfs_realtime_pb2.FeedHeader.FULL_DATASET) with self._get_producer() as producer: log.info('Publishing full feed...') producer.publish(feed.SerializeToString(), routing_key=task.load_realtime.queue_name) log.info('Full feed published.') finally: db.session.remove()
def callback(body, message): task = task_pb2.Task() try: # `body` is of unicode type, but we need str type for # `ParseFromString()` to work. It seems to work. # Maybe kombu estimate that, without any information, # the body should be something as json, and thus a # unicode string. On the c++ side, I didn't manage to # find a way to give a content-type or something like # that. body = str(body) task.ParseFromString(body) except DecodeError as e: log.warn('invalid protobuf: {}'.format(str(e))) return log.info('getting a request: {}'.format(task)) if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime: return feed = convert_to_gtfsrt(RealTimeUpdate.all(task.load_realtime.contributors)) with self._get_producer() as producer: producer.publish(feed.SerializeToString(), routing_key=task.load_realtime.queue_name)
def test_populate_pb_with_one_stop_time(): """ an easy one: we have one vj with only one stop time updated fill protobuf from trip_update Verify protobuf """ navitia_vj = {'id': 'vehicle_journey:1', 'stop_times': [ {'arrival_time': None, 'departure_time': datetime.time(8, 10), 'stop_point': {'id': 'sa:1'}}, {'arrival_time': datetime.time(9, 10), 'departure_time': None, 'stop_point': {'id': 'sa:2'}} ]} with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8)) trip_update.vj = vj st = StopTimeUpdate({'id': 'sa:1'}, departure=_dt("8:15"), arrival=None) real_time_update = RealTimeUpdate(raw_data=None, connector='ire') real_time_update.trip_updates.append(trip_update) trip_update.stop_time_updates.append(st) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version == '1' pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED pb_stop_time = feed_entity.entity[0].trip_update.stop_time_update[0] assert pb_stop_time.arrival.time == 0 assert pb_stop_time.departure.time == to_posix_time(_dt("8:15")) assert pb_stop_time.stop_id == 'sa:1'
def test_populate_pb_with_cancelation(): """ VJ cancelation """ navitia_vj = {'trip': {'id': 'vehicle_journey:1'}} with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8)) trip_update.vj = vj trip_update.status = 'delete' trip_update.message = 'Message Test' real_time_update = RealTimeUpdate(raw_data=None, connector='ire') trip_update.contributor = 'kisio-digital' real_time_update.trip_updates.append(trip_update) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version == '1' pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == True assert pb_trip_update.Extensions[ kirin_pb2.trip_message] == 'Message Test' assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == True assert pb_trip_update.trip.Extensions[ kirin_pb2.contributor] == 'kisio-digital' assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_deleted_stop_time(): """ test protobuf for partial delete nav vj sa:1 * ---- * sa:2 * ---- * sa:3 * ---- * sa:4 we stop the vj from going to sa:3 and sa:4 sa:1 * ---- * sa:2 And we delay sa:1 of 5 minutes Note: in the message sent sa:2 departure will be removed as well as sa:3 departures/arrival and sa:4' arrival """ #we add another impacted stop time to the Model from datetime import time navitia_vj = { 'trip': { 'id': 'vehicle_journey:1' }, 'stop_times': [{ 'arrival_time': None, 'departure_time': time(8, 11), 'stop_point': { 'id': 'sa:1', 'stop_area': { 'timezone': 'Europe/Paris' } } }, { 'arrival_time': time(9, 10), 'departure_time': time(9, 11), 'stop_point': { 'id': 'sa:2', 'stop_area': { 'timezone': 'Europe/Paris' } } }, { 'arrival_time': time(10, 10), 'departure_time': time(10, 11), 'stop_point': { 'id': 'sa:3', 'stop_area': { 'timezone': 'Europe/Paris' } } }, { 'arrival_time': time(11, 10), 'departure_time': None, 'stop_point': { 'id': 'sa:4', 'stop_area': { 'timezone': 'Europe/Paris' } } }] } with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8)) trip_update.vj = vj real_time_update = RealTimeUpdate(raw_data=None, connector='ire', contributor='realtime.ire') real_time_update.trip_updates.append(trip_update) st = StopTimeUpdate({'id': 'sa:1'}, departure=_dt("8:15"), departure_delay=timedelta(minutes=5), arrival=None) trip_update.stop_time_updates.append(st) st = StopTimeUpdate( {'id': 'sa:2'}, arrival=_dt("9:10"), departure=_dt("9:11"), dep_status='delete', # we delete the departure message="bob's on the track") real_time_update.trip_updates[0].stop_time_updates.append(st) st = StopTimeUpdate( {'id': 'sa:3'}, # Note: we still send the departure/arrival for coherence arrival=_dt("10:10"), dep_status='delete', # we delete both departure and arrival departure=_dt("10:11"), arr_status='delete', message="bob's on the track") real_time_update.trip_updates[0].stop_time_updates.append(st) st = StopTimeUpdate( {'id': 'sa:4'}, arrival=_dt("11:10"), arr_status='delete', # we delete only the arrival departure=_dt("11:11"), message="bob's on the track") real_time_update.trip_updates[0].stop_time_updates.append(st) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version, '1' assert len(feed_entity.entity) == 1 pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is False assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED assert len(pb_trip_update.stop_time_update) == 4 pb_stop_time = pb_trip_update.stop_time_update[0] assert pb_stop_time.stop_id == 'sa:1' assert pb_stop_time.arrival.time == 0 assert pb_stop_time.arrival.delay == 0 assert pb_stop_time.departure.time == to_posix_time(_dt("8:15")) assert pb_stop_time.departure.delay == 5 * 60 assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == "" pb_stop_time = pb_trip_update.stop_time_update[1] assert pb_stop_time.stop_id == 'sa:2' assert pb_stop_time.arrival.time == to_posix_time(_dt("9:10")) assert pb_stop_time.arrival.delay == 0 # the arrival at the stop is still scheduled, but the departure is skipped assert pb_stop_time.arrival.Extensions[kirin_pb2.stop_time_event_relationship] == \ gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SCHEDULED assert pb_stop_time.departure.time == to_posix_time(_dt("9:11")) assert pb_stop_time.departure.delay == 0 assert pb_stop_time.departure.Extensions[kirin_pb2.stop_time_event_relationship] == \ gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED assert pb_stop_time.Extensions[ kirin_pb2.stoptime_message] == "bob's on the track" pb_stop_time = pb_trip_update.stop_time_update[2] assert pb_stop_time.stop_id == 'sa:3' # both departure and arrival are SKIPPED assert pb_stop_time.arrival.time == to_posix_time(_dt("10:10")) assert pb_stop_time.arrival.delay == 0 assert pb_stop_time.arrival.Extensions[kirin_pb2.stop_time_event_relationship] == \ gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED assert pb_stop_time.departure.time == to_posix_time(_dt("10:11")) assert pb_stop_time.departure.delay == 0 assert pb_stop_time.departure.Extensions[kirin_pb2.stop_time_event_relationship] == \ gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED assert pb_stop_time.Extensions[ kirin_pb2.stoptime_message] == "bob's on the track" pb_stop_time = pb_trip_update.stop_time_update[3] assert pb_stop_time.stop_id == 'sa:4' # only the arrival is skipped, the departure does not exists as it's the last stop assert pb_stop_time.arrival.time == to_posix_time(_dt("11:10")) assert pb_stop_time.arrival.delay == 0 assert pb_stop_time.arrival.Extensions[kirin_pb2.stop_time_event_relationship] == \ gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED assert pb_stop_time.departure.time == to_posix_time(_dt("11:11")) assert pb_stop_time.departure.delay == 0 assert pb_stop_time.departure.Extensions[kirin_pb2.stop_time_event_relationship] == \ gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SCHEDULED assert pb_stop_time.Extensions[ kirin_pb2.stoptime_message] == "bob's on the track"
def test_populate_pb_with_two_stop_time(): """ an easy one: we have one vj with only one stop time updated fill protobuf from trip_update Verify protobuf """ #we add another impacted stop time to the Model navitia_vj = { 'trip': { 'id': 'vehicle_journey:1' }, 'stop_times': [{ 'arrival_time': None, 'departure_time': datetime.time(8, 10), 'stop_point': { 'id': 'sa:1', 'stop_area': { 'timezone': 'Europe/Paris' } } }, { 'arrival_time': datetime.time(9, 10), 'departure_time': None, 'stop_point': { 'id': 'sa:2', 'stop_area': { 'timezone': 'Europe/Paris' } } }] } with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8)) trip_update.vj = vj real_time_update = RealTimeUpdate(raw_data=None, connector='ire', contributor='realtime.ire') real_time_update.trip_updates.append(trip_update) st = StopTimeUpdate({'id': 'sa:1'}, departure=_dt("8:15"), departure_delay=timedelta(minutes=5), arrival=None) trip_update.stop_time_updates.append(st) st = StopTimeUpdate({'id': 'sa:2'}, departure=_dt("8:21"), departure_delay=timedelta(minutes=-40), arrival=_dt("8:20"), arrival_delay=timedelta(minutes=-40), message="bob's on the track") real_time_update.trip_updates[0].stop_time_updates.append(st) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version, '1' assert len(feed_entity.entity) == 1 pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is False assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED assert len(pb_trip_update.stop_time_update) == 2 pb_stop_time = pb_trip_update.stop_time_update[0] assert pb_stop_time.stop_id == 'sa:1' assert pb_stop_time.arrival.time == 0 assert pb_stop_time.arrival.delay == 0 assert pb_stop_time.departure.time == to_posix_time(_dt("8:15")) assert pb_stop_time.departure.delay == 5 * 60 assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == "" pb_stop_time = pb_trip_update.stop_time_update[1] assert pb_stop_time.stop_id == 'sa:2' assert pb_stop_time.arrival.time == to_posix_time(_dt("8:20")) assert pb_stop_time.arrival.delay == -40 * 60 assert pb_stop_time.departure.time == to_posix_time(_dt("8:21")) assert pb_stop_time.departure.delay == -40 * 60 assert pb_stop_time.Extensions[ kirin_pb2.stoptime_message] == "bob's on the track"
def test_populate_pb_with_deleted_stop_time(): """ test protobuf for partial delete nav vj sa:1 * ---- * sa:2 * ---- * sa:3 * ---- * sa:4 we stop the vj from going to sa:3 and sa:4 sa:1 * ---- * sa:2 And we delay sa:1 of 5 minutes Note: in the message sent sa:2 departure will be removed as well as sa:3 departures/arrival and sa:4' arrival """ # we add another impacted stop time to the Model from datetime import time navitia_vj = { "trip": { "id": "vehicle_journey:1" }, "stop_times": [ { "utc_arrival_time": None, "utc_departure_time": time(6, 11), "stop_point": { "id": "sa:1", "stop_area": { "timezone": "Europe/Paris" } }, }, { "utc_arrival_time": time(7, 10), "utc_departure_time": time(7, 11), "stop_point": { "id": "sa:2", "stop_area": { "timezone": "Europe/Paris" } }, }, { "utc_arrival_time": time(8, 10), "utc_departure_time": time(8, 11), "stop_point": { "id": "sa:3", "stop_area": { "timezone": "Europe/Paris" } }, }, { "utc_arrival_time": time(9, 10), "utc_departure_time": None, "stop_point": { "id": "sa:4", "stop_area": { "timezone": "Europe/Paris" } }, }, ], } with app.app_context(): vj = VehicleJourney(navitia_vj, datetime.datetime(2015, 9, 8, 5, 11, 0), datetime.datetime(2015, 9, 8, 10, 10, 0)) trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) real_time_update.trip_updates.append(trip_update) st = StopTimeUpdate({"id": "sa:1"}, departure=_dt("8:15"), departure_delay=timedelta(minutes=5), arrival=None) trip_update.stop_time_updates.append(st) st = StopTimeUpdate( {"id": "sa:2"}, arrival=_dt("9:10"), departure=_dt("9:11"), dep_status="delete", # we delete the departure message="bob's on the track", ) real_time_update.trip_updates[0].stop_time_updates.append(st) st = StopTimeUpdate( {"id": "sa:3"}, # Note: we still send the departure/arrival for coherence arrival=_dt("10:10"), dep_status="delete", # we delete both departure and arrival departure=_dt("10:11"), arr_status="delete", message="bob's on the track", ) real_time_update.trip_updates[0].stop_time_updates.append(st) st = StopTimeUpdate( {"id": "sa:4"}, arrival=_dt("11:10"), arr_status="delete", # we delete only the arrival departure=_dt("11:11"), message="bob's on the track", ) real_time_update.trip_updates[0].stop_time_updates.append(st) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version, "1" assert len(feed_entity.entity) == 1 pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == "vehicle_journey:1" assert pb_trip_update.trip.start_date == "20150908" assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True assert pb_trip_update.trip.HasExtension(kirin_pb2.company_id) is False assert pb_trip_update.HasExtension(kirin_pb2.effect) is False assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED assert len(pb_trip_update.stop_time_update) == 4 pb_stop_time = pb_trip_update.stop_time_update[0] assert pb_stop_time.stop_id == "sa:1" assert pb_stop_time.arrival.time == 0 assert pb_stop_time.arrival.delay == 0 assert pb_stop_time.departure.time == to_posix_time(_dt("8:15")) assert pb_stop_time.departure.delay == 5 * 60 assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == "" pb_stop_time = pb_trip_update.stop_time_update[1] assert pb_stop_time.stop_id == "sa:2" assert pb_stop_time.arrival.time == to_posix_time(_dt("9:10")) assert pb_stop_time.arrival.delay == 0 # the arrival at the stop is still scheduled, but the departure is skipped assert (pb_stop_time.arrival.Extensions[ kirin_pb2.stop_time_event_relationship] == gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SCHEDULED) assert pb_stop_time.departure.time == to_posix_time(_dt("9:11")) assert pb_stop_time.departure.delay == 0 assert (pb_stop_time.departure.Extensions[ kirin_pb2.stop_time_event_relationship] == gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED) assert pb_stop_time.Extensions[ kirin_pb2.stoptime_message] == "bob's on the track" pb_stop_time = pb_trip_update.stop_time_update[2] assert pb_stop_time.stop_id == "sa:3" # both departure and arrival are SKIPPED assert pb_stop_time.arrival.time == to_posix_time(_dt("10:10")) assert pb_stop_time.arrival.delay == 0 assert (pb_stop_time.arrival.Extensions[ kirin_pb2.stop_time_event_relationship] == gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED) assert pb_stop_time.departure.time == to_posix_time(_dt("10:11")) assert pb_stop_time.departure.delay == 0 assert (pb_stop_time.departure.Extensions[ kirin_pb2.stop_time_event_relationship] == gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED) assert pb_stop_time.Extensions[ kirin_pb2.stoptime_message] == "bob's on the track" pb_stop_time = pb_trip_update.stop_time_update[3] assert pb_stop_time.stop_id == "sa:4" # only the arrival is skipped, the departure does not exists as it's the last stop assert pb_stop_time.arrival.time == to_posix_time(_dt("11:10")) assert pb_stop_time.arrival.delay == 0 assert (pb_stop_time.arrival.Extensions[ kirin_pb2.stop_time_event_relationship] == gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED) assert pb_stop_time.departure.time == to_posix_time(_dt("11:11")) assert pb_stop_time.departure.delay == 0 assert (pb_stop_time.departure.Extensions[ kirin_pb2.stop_time_event_relationship] == gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SCHEDULED) assert pb_stop_time.Extensions[ kirin_pb2.stoptime_message] == "bob's on the track"
def test_populate_pb_with_one_stop_time(): """ an easy one: we have one vj with only one stop time updated fill protobuf from trip_update Verify protobuf """ navitia_vj = { "trip": { "id": "vehicle_journey:1" }, "stop_times": [ { "utc_arrival_time": None, "utc_departure_time": datetime.time(6, 10), "stop_point": { "id": "sa:1", "stop_area": { "timezone": "Europe/Paris" } }, }, { "utc_arrival_time": datetime.time(7, 10), "utc_departure_time": None, "stop_point": { "id": "sa:2", "stop_area": { "timezone": "Europe/Paris" } }, }, ], } with app.app_context(): vj = VehicleJourney(navitia_vj, datetime.datetime(2015, 9, 8, 5, 10, 0), datetime.datetime(2015, 9, 8, 8, 10, 0)) trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID) st = StopTimeUpdate({"id": "sa:1"}, departure=_dt("8:15"), arrival=None) real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) real_time_update.trip_updates.append(trip_update) trip_update.stop_time_updates.append(st) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version == "1" pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == "vehicle_journey:1" assert pb_trip_update.trip.start_date == "20150908" assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED pb_stop_time = feed_entity.entity[0].trip_update.stop_time_update[0] assert pb_stop_time.arrival.time == 0 assert pb_stop_time.departure.time == to_posix_time(_dt("8:15")) assert pb_stop_time.stop_id == "sa:1" assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True assert pb_trip_update.trip.HasExtension(kirin_pb2.company_id) is False assert pb_trip_update.HasExtension(kirin_pb2.effect) is False
def test_populate_pb_for_added_trip(): """ For an added trip we don't call navitia to get a vj instead we create and initialize one vj only with 'id' Fill protobuf from trip_update Verify protobuf """ navitia_vj = make_navitia_empty_vj('vehicle_journey:1') with app.app_context(): trip_update = TripUpdate() vj = VehicleJourney( navitia_vj, utc_since_dt=utc.localize(datetime.datetime(2015, 9, 8, 5, 10, 0)), utc_until_dt=utc.localize(datetime.datetime(2015, 9, 8, 8, 10, 0)), vj_start_dt=utc.localize(datetime.datetime(2015, 9, 8, 5, 10, 0))) trip_update.vj = vj trip_update.status = 'add' trip_update.effect = 'ADDITIONAL_SERVICE' trip_update.physical_mode_id = 'physical_mode:LongDistanceTrain' real_time_update = RealTimeUpdate(raw_data=None, connector='cots', contributor='realtime.cots') real_time_update.trip_updates.append(trip_update) st = StopTimeUpdate({'id': 'sa:1'}, departure=_dt("8:15"), departure_delay=timedelta(minutes=5), arrival=None, arr_status='none', dep_status='add') trip_update.stop_time_updates.append(st) st = StopTimeUpdate({'id': 'sa:2'}, departure=_dt("8:21"), departure_delay=timedelta(minutes=-40), arrival=_dt("8:20"), arrival_delay=timedelta(minutes=-40), message="bob's on the track", arr_status='add', dep_status='none') trip_update.stop_time_updates.append(st) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version, '1' assert len(feed_entity.entity) == 1 pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == 'OCE:SN:vehicle_journey:1' assert pb_trip_update.trip.start_date == '20150908' assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is False assert pb_trip_update.trip.HasExtension(kirin_pb2.company_id) is False assert pb_trip_update.HasExtension(kirin_pb2.effect) is True assert pb_trip_update.Extensions[ kirin_pb2.effect] == gtfs_realtime_pb2.Alert.ADDITIONAL_SERVICE assert pb_trip_update.vehicle.Extensions[ kirin_pb2.physical_mode_id] == 'physical_mode:LongDistanceTrain' assert len(pb_trip_update.stop_time_update) == 2 pb_stop_time = pb_trip_update.stop_time_update[0] assert pb_stop_time.stop_id == 'sa:1' assert pb_stop_time.arrival.time == 0 assert pb_stop_time.arrival.delay == 0 assert pb_stop_time.departure.time == to_posix_time(_dt("8:15")) assert pb_stop_time.departure.delay == 5 * 60 assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == "" assert pb_stop_time.arrival.Extensions[ kirin_pb2.stop_time_event_status] == kirin_pb2.SCHEDULED assert pb_stop_time.departure.Extensions[ kirin_pb2.stop_time_event_status] == kirin_pb2.ADDED pb_stop_time = pb_trip_update.stop_time_update[1] assert pb_stop_time.stop_id == 'sa:2' assert pb_stop_time.arrival.time == to_posix_time(_dt("8:20")) assert pb_stop_time.arrival.delay == -40 * 60 assert pb_stop_time.departure.time == to_posix_time(_dt("8:21")) assert pb_stop_time.departure.delay == -40 * 60 assert pb_stop_time.Extensions[ kirin_pb2.stoptime_message] == "bob's on the track" assert pb_stop_time.arrival.Extensions[ kirin_pb2.stop_time_event_status] == kirin_pb2.ADDED assert pb_stop_time.departure.Extensions[ kirin_pb2.stop_time_event_status] == kirin_pb2.SCHEDULED
def test_populate_pb_for_added_trip(): """ For an added trip we don't call navitia to get a vj instead we create and initialize one vj only with 'id' Fill protobuf from trip_update Verify protobuf """ navitia_vj = make_navitia_empty_vj("vehicle_journey:1") with app.app_context(): vj = VehicleJourney( navitia_vj, since_dt=datetime.datetime(2015, 9, 8, 5, 10, 0), until_dt=datetime.datetime(2015, 9, 8, 8, 10, 0), vj_start_dt=datetime.datetime(2015, 9, 8, 5, 10, 0), ) trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID) trip_update.vj = vj trip_update.status = "add" trip_update.effect = "ADDITIONAL_SERVICE" trip_update.physical_mode_id = "physical_mode:LongDistanceTrain" real_time_update = make_rt_update( raw_data=None, connector_type=ConnectorType.cots.value, contributor_id=COTS_CONTRIBUTOR_ID) real_time_update.trip_updates.append(trip_update) st = StopTimeUpdate( {"id": "sa:1"}, departure=_dt("8:15"), departure_delay=timedelta(minutes=5), arrival=None, arr_status="none", dep_status="add", ) trip_update.stop_time_updates.append(st) st = StopTimeUpdate( {"id": "sa:2"}, departure=_dt("8:21"), departure_delay=timedelta(minutes=-40), arrival=_dt("8:20"), arrival_delay=timedelta(minutes=-40), message="bob's on the track", arr_status="add", dep_status="none", ) trip_update.stop_time_updates.append(st) db.session.add(real_time_update) db.session.commit() feed_entity = convert_to_gtfsrt(real_time_update.trip_updates) assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL assert feed_entity.header.gtfs_realtime_version, "1" assert len(feed_entity.entity) == 1 pb_trip_update = feed_entity.entity[0].trip_update assert pb_trip_update.trip.trip_id == "OCE:SN:vehicle_journey:1" assert pb_trip_update.trip.start_date == "20150908" assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True assert pb_trip_update.trip.HasExtension(kirin_pb2.company_id) is False assert pb_trip_update.HasExtension(kirin_pb2.effect) is True assert pb_trip_update.Extensions[ kirin_pb2.effect] == gtfs_realtime_pb2.Alert.ADDITIONAL_SERVICE assert pb_trip_update.vehicle.Extensions[ kirin_pb2.physical_mode_id] == "physical_mode:LongDistanceTrain" assert len(pb_trip_update.stop_time_update) == 2 pb_stop_time = pb_trip_update.stop_time_update[0] assert pb_stop_time.stop_id == "sa:1" assert pb_stop_time.arrival.time == 0 assert pb_stop_time.arrival.delay == 0 assert pb_stop_time.departure.time == to_posix_time(_dt("8:15")) assert pb_stop_time.departure.delay == 5 * 60 assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == "" assert pb_stop_time.arrival.Extensions[ kirin_pb2.stop_time_event_status] == kirin_pb2.SCHEDULED assert pb_stop_time.departure.Extensions[ kirin_pb2.stop_time_event_status] == kirin_pb2.ADDED pb_stop_time = pb_trip_update.stop_time_update[1] assert pb_stop_time.stop_id == "sa:2" assert pb_stop_time.arrival.time == to_posix_time(_dt("8:20")) assert pb_stop_time.arrival.delay == -40 * 60 assert pb_stop_time.departure.time == to_posix_time(_dt("8:21")) assert pb_stop_time.departure.delay == -40 * 60 assert pb_stop_time.Extensions[ kirin_pb2.stoptime_message] == "bob's on the track" assert pb_stop_time.arrival.Extensions[ kirin_pb2.stop_time_event_status] == kirin_pb2.ADDED assert pb_stop_time.departure.Extensions[ kirin_pb2.stop_time_event_status] == kirin_pb2.SCHEDULED
def _on_request(self, message): log = logging.getLogger(__name__) status = "OK" log_dict = {} start_datetime = datetime.utcnow() try: task = task_pb2.Task() try: # `body` is a string, but we need binary type for # `ParseFromString()` to work. It seems to work. body = bytes(message.payload, encoding="utf-8") task.ParseFromString(body) except DecodeError as e: log.warning("invalid protobuf: {}".format(str(e))) return log.info("Getting a full feed publication request", extra={"task": task}) if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime: return begin_date = None end_date = None if hasattr(task.load_realtime, "begin_date") and task.load_realtime.begin_date: begin_date = str_to_date(task.load_realtime.begin_date) if hasattr(task.load_realtime, "end_date") and task.load_realtime.end_date: end_date = str_to_date(task.load_realtime.end_date) feed = convert_to_gtfsrt( TripUpdate.find_by_contributor_period( task.load_realtime.contributors, begin_date, end_date), gtfs_realtime_pb2.FeedHeader.FULL_DATASET, ) feed_str = feed.SerializeToString() log_dict.update({ "contributors": task.load_realtime.contributors, "routing_key": task.load_realtime.queue_name, "output_trip_update_count": len(feed.entity), "output_feed_size": sys.getsizeof(feed_str), }) log.info( "Starting of full feed publication {}, {}".format( len(feed_str), task), extra={ "size": len(feed_str), "task": task }, ) record_custom_parameter("contributors", task.load_realtime.contributors) # http://docs.celeryproject.org/projects/kombu/en/latest/userguide/producers.html#bypassing-routing-by-using-the-anon-exchange self.producer.publish( feed_str, routing_key=task.load_realtime.queue_name, retry=True, retry_policy={ "interval_start": 0, # First retry immediately, "interval_step": 2, # then increase by 2s for every retry. "interval_max": 10, # but don't exceed 10s between retries. "max_retries": self.max_retries, # give up after 10 (by default) tries. }, ) except Exception as e: status = "failure" log_dict.update({"reason": str(e)}) record_custom_parameter("reason", str(e)) finally: duration = (datetime.utcnow() - start_datetime).total_seconds() log_dict.update({"duration": duration}) record_call("kirin_reload_info", status, **log_dict) log.info("End of full feed publication", extra={ "duration": duration, "task": task }) db.session.remove() return log_dict