def handle(real_time_update, trip_updates):
"""
receive a RealTimeUpdate with at least one TripUpdate filled with the data received
by the connector. each TripUpdate is associated with the VehicleJourney returned by jormugandr
"""
if not real_time_update:
raise TypeError()
for trip_update in trip_updates:
#find if there already a row in db
old = TripUpdate.find_by_dated_vj(trip_update.vj.navitia_id, trip_update.vj.circulation_date)
#merge the theoric, the current realtime, and the new relatime
current_trip_update = merge(trip_update, old)
# we have to link the current_vj_update with the new real_time_update
# this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy
current_trip_update.real_time_updates.append(real_time_update)
persist(real_time_update)
feed = convert_to_gtfsrt([real_time_update])
publish(feed, real_time_update)
return real_time_update
def handle(real_time_update, trip_updates, contributor):
"""
receive a RealTimeUpdate with at least one TripUpdate filled with the data received
by the connector. each TripUpdate is associated with the VehicleJourney returned by jormugandr
"""
if not real_time_update:
raise TypeError()
for trip_update in trip_updates:
# find if there already a row in db
old = TripUpdate.find_by_dated_vj(trip_update.vj.navitia_trip_id, trip_update.vj.circulation_date)
# merge the theoric, the current realtime, and the new realtime
current_trip_update = merge(trip_update.vj.navitia_vj, old, trip_update)
# manage and adjust consistency if possible
if manage_consistency(current_trip_update):
# we have to link the current_vj_update with the new real_time_update
# this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy
current_trip_update.real_time_updates.append(real_time_update)
persist(real_time_update)
feed = convert_to_gtfsrt(real_time_update.trip_updates)
publish(feed, contributor)
return real_time_update
def test_populate_pb_with_cancelation():
"""
VJ cancelation
"""
navitia_vj = {'id': 'vehicle_journey:1'}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8))
trip_update.vj = vj
trip_update.status = 'delete'
real_time_update = RealTimeUpdate(raw_data=None, connector='ire')
real_time_update.trip_updates.append(trip_update)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version == '1'
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED
assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_full_dataset():
"""
VJ cancelation
"""
navitia_vj = {'trip': {'id': 'vehicle_journey:1'}}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8))
trip_update.vj = vj
trip_update.status = 'delete'
trip_update.message = 'Message Test'
real_time_update = RealTimeUpdate(raw_data=None, connector='ire')
trip_update.contributor = 'kisio-digital'
real_time_update.trip_updates.append(trip_update)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates, gtfs_realtime_pb2.FeedHeader.FULL_DATASET)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.FULL_DATASET
assert feed_entity.header.gtfs_realtime_version == '1'
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == True
assert pb_trip_update.Extensions[kirin_pb2.trip_message] == 'Message Test'
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == True
assert pb_trip_update.trip.Extensions[kirin_pb2.contributor] == 'kisio-digital'
assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def handle(builder, real_time_update, trip_updates):
"""
Receive a RealTimeUpdate with at least one TripUpdate filled with the data received
by the connector.
Each TripUpdate is associated with the base-schedule VehicleJourney, complete/merge realtime is done using builder
Then persist in db, publish for Navitia
Returns real_time_update and the log_dict
"""
if not real_time_update:
raise TypeError()
id_timestamp_tuples = [(tu.vj.navitia_trip_id, tu.vj.start_timestamp)
for tu in trip_updates]
old_trip_updates = TripUpdate.find_by_dated_vjs(id_timestamp_tuples)
for trip_update in trip_updates:
# find if there is already a row in db
old = next(
(tu for tu in old_trip_updates
if tu.vj.navitia_trip_id == trip_update.vj.navitia_trip_id
and tu.vj.start_timestamp == trip_update.vj.start_timestamp),
None,
)
# merge the base schedule, the current realtime, and the new realtime
current_trip_update = builder.merge_trip_updates(
trip_update.vj.navitia_vj, old, trip_update)
# manage and adjust consistency if possible
if current_trip_update is not None and check_consistency(
current_trip_update):
# we have to link the current_vj_update with the new real_time_update
# this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy
current_trip_update.real_time_updates.append(real_time_update)
db_commit(real_time_update)
feed = convert_to_gtfsrt(real_time_update.trip_updates,
gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL)
feed_str = feed.SerializeToString()
publish(feed_str, builder.contributor.id)
data_time = datetime.datetime.utcfromtimestamp(feed.header.timestamp)
log_dict = {
"contributor": builder.contributor.id,
"timestamp": data_time,
"output_trip_update_count": len(feed.entity),
"output_feed_size": sys.getsizeof(feed_str),
}
# After merging trip_updates information of connector realtime, navitia and kirin database, if there is no new
# information destined to navitia, update real_time_update with status = 'KO' and a proper error message.
if not real_time_update.trip_updates and real_time_update.status == "pending":
msg = "No new information destined for navitia on {}".format(
real_time_update.contributor_id)
set_rtu_status_ko(real_time_update,
msg,
is_reprocess_same_data_allowed=False)
logging.getLogger(__name__).warning("RealTimeUpdate id={}: {}".format(
real_time_update.id, msg),
extra=log_dict)
db_commit(real_time_update)
return real_time_update, log_dict
def handle(real_time_update, trip_updates, contributor, is_new_complete=False):
"""
receive a RealTimeUpdate with at least one TripUpdate filled with the data received
by the connector. each TripUpdate is associated with the VehicleJourney returned by jormugandr
Returns real_time_update and the log_dict
"""
if not real_time_update:
raise TypeError()
id_timestamp_tuples = [(tu.vj.navitia_trip_id, tu.vj.get_start_timestamp()) for tu in trip_updates]
old_trip_updates = TripUpdate.find_by_dated_vjs(id_timestamp_tuples)
for trip_update in trip_updates:
# find if there is already a row in db
old = next((tu for tu in old_trip_updates if tu.vj.navitia_trip_id == trip_update.vj.navitia_trip_id
and tu.vj.get_start_timestamp() == trip_update.vj.get_start_timestamp()), None)
# merge the base schedule, the current realtime, and the new realtime
current_trip_update = merge(trip_update.vj.navitia_vj, old, trip_update, is_new_complete=is_new_complete)
# manage and adjust consistency if possible
if current_trip_update and manage_consistency(current_trip_update):
# we have to link the current_vj_update with the new real_time_update
# this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy
current_trip_update.real_time_updates.append(real_time_update)
persist(real_time_update)
feed = convert_to_gtfsrt(real_time_update.trip_updates)
feed_str = feed.SerializeToString()
publish(feed_str, contributor)
data_time = datetime.datetime.utcfromtimestamp(feed.header.timestamp)
log_dict = {'contributor': contributor, 'timestamp': data_time, 'trip_update_count': len(feed.entity),
'size': len(feed_str)}
return real_time_update, log_dict
def test_populate_pb_with_cancelation():
"""
VJ cancelation
"""
navitia_vj = {
'trip': {
'id': 'vehicle_journey:1'
},
'stop_times': [{
'utc_arrival_time': datetime.time(8, 10),
'stop_point': {
'stop_area': {
'timezone': 'UTC'
}
}
}]
}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(
navitia_vj, utc.localize(datetime.datetime(2015, 9, 8, 7, 10, 0)),
utc.localize(datetime.datetime(2015, 9, 8, 11, 5, 0)))
trip_update.vj = vj
trip_update.status = 'delete'
trip_update.message = 'Message Test'
real_time_update = RealTimeUpdate(raw_data=None,
connector='cots',
contributor='realtime.cots')
trip_update.contributor = 'kisio-digital'
trip_update.company_id = 'sncf'
trip_update.effect = 'REDUCED_SERVICE'
real_time_update.trip_updates.append(trip_update)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version == '1'
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == True
assert pb_trip_update.Extensions[
kirin_pb2.trip_message] == 'Message Test'
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == True
assert pb_trip_update.trip.Extensions[
kirin_pb2.contributor] == 'kisio-digital'
assert pb_trip_update.trip.Extensions[kirin_pb2.company_id] == 'sncf'
assert pb_trip_update.Extensions[
kirin_pb2.effect] == gtfs_realtime_pb2.Alert.REDUCED_SERVICE
assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_cancelation():
"""
VJ cancelation
"""
navitia_vj = {
"trip": {
"id": "vehicle_journey:1"
},
"stop_times": [{
"utc_arrival_time": datetime.time(8, 10),
"stop_point": {
"stop_area": {
"timezone": "UTC"
}
}
}],
}
with app.app_context():
vj = VehicleJourney(navitia_vj,
datetime.datetime(2015, 9, 8, 7, 10, 0),
datetime.datetime(2015, 9, 8, 11, 5, 0))
trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.vj = vj
trip_update.status = "delete"
trip_update.message = "Message Test"
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.company_id = "sncf"
trip_update.effect = "REDUCED_SERVICE"
real_time_update.trip_updates.append(trip_update)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version == "1"
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == "vehicle_journey:1"
assert pb_trip_update.trip.start_date == "20150908"
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is True
assert pb_trip_update.Extensions[
kirin_pb2.trip_message] == "Message Test"
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True
assert pb_trip_update.trip.Extensions[
kirin_pb2.contributor] == COTS_CONTRIBUTOR_ID
assert pb_trip_update.trip.Extensions[kirin_pb2.company_id] == "sncf"
assert pb_trip_update.Extensions[
kirin_pb2.effect] == gtfs_realtime_pb2.Alert.REDUCED_SERVICE
assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_full_dataset():
"""
VJ cancelation
"""
navitia_vj = {
'trip': {
'id': 'vehicle_journey:1'
},
'stop_times': [{
'arrival_time': datetime.time(8, 10),
'stop_point': {
'stop_area': {
'timezone': 'UTC'
}
}
}]
}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8))
trip_update.vj = vj
trip_update.status = 'delete'
trip_update.message = 'Message Test'
real_time_update = RealTimeUpdate(raw_data=None,
connector='ire',
contributor='realtime.ire')
trip_update.contributor = 'kisio-digital'
trip_update.company_id = 'keolis'
trip_update.effect = 'DETOUR'
real_time_update.trip_updates.append(trip_update)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(
real_time_update.trip_updates,
gtfs_realtime_pb2.FeedHeader.FULL_DATASET)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.FULL_DATASET
assert feed_entity.header.gtfs_realtime_version == '1'
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == True
assert pb_trip_update.Extensions[
kirin_pb2.trip_message] == 'Message Test'
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == True
assert pb_trip_update.trip.Extensions[
kirin_pb2.contributor] == 'kisio-digital'
assert pb_trip_update.trip.Extensions[kirin_pb2.company_id] == 'keolis'
assert pb_trip_update.Extensions[
kirin_pb2.effect] == gtfs_realtime_pb2.Alert.DETOUR
assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_two_stop_time():
"""
an easy one: we have one vj with only one stop time updated
fill protobuf from trip_update
Verify protobuf
"""
#we add another impacted stop time to the Model
navitia_vj = {'trip': {'id': 'vehicle_journey:1'}, 'stop_times': [
{'arrival_time': None, 'departure_time': datetime.time(8, 10), 'stop_point': {'id': 'sa:1'}},
{'arrival_time': datetime.time(9, 10), 'departure_time': None, 'stop_point': {'id': 'sa:2'}}
]}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8))
trip_update.vj = vj
st = StopTimeUpdate({'id': 'sa:1'}, departure=_dt("8:15"), arrival=None)
real_time_update = RealTimeUpdate(raw_data=None, connector='ire')
real_time_update.trip_updates.append(trip_update)
trip_update.stop_time_updates.append(st)
st = StopTimeUpdate({'id': 'sa:2'},
departure=_dt("8:21"),
arrival=_dt("8:20"),
message="bob's on the track")
real_time_update.trip_updates[0].stop_time_updates.append(st)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version, '1'
assert len(feed_entity.entity) == 1
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == False
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == False
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED
assert len(pb_trip_update.stop_time_update) == 2
pb_stop_time = pb_trip_update.stop_time_update[0]
assert pb_stop_time.stop_id == 'sa:1'
assert pb_stop_time.arrival.time == 0
assert pb_stop_time.departure.time == to_posix_time(_dt("8:15"))
assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == ""
pb_stop_time = pb_trip_update.stop_time_update[1]
assert pb_stop_time.stop_id == 'sa:2'
assert pb_stop_time.arrival.time == to_posix_time(_dt("8:20"))
assert pb_stop_time.departure.time == to_posix_time(_dt("8:21"))
assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == "bob's on the track"
def _on_request(self, message):
log = logging.getLogger(__name__)
try:
task = task_pb2.Task()
try:
# `body` is of unicode type, but we need str type for
# `ParseFromString()` to work. It seems to work.
# Maybe kombu estimate that, without any information,
# the body should be something as json, and thus a
# unicode string. On the c++ side, I didn't manage to
# find a way to give a content-type or something like
# that.
body = str(message.payload)
task.ParseFromString(body)
except DecodeError as e:
log.warn('invalid protobuf: {}'.format(str(e)))
return
log.info('Getting a full feed publication request', extra={'task': task})
if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime:
return
start_datetime = datetime.utcnow()
begin_date = None
end_date = None
if hasattr(task.load_realtime, "begin_date"):
if task.load_realtime.begin_date:
begin_date = str_to_date(task.load_realtime.begin_date)
if hasattr(task.load_realtime, "end_date"):
if task.load_realtime.end_date:
end_date = str_to_date(task.load_realtime.end_date)
feed = convert_to_gtfsrt(TripUpdate.find_by_contributor_period(task.load_realtime.contributors,
begin_date,
end_date),
gtfs_realtime_pb2.FeedHeader.FULL_DATASET)
feed_str = feed.SerializeToString()
log.info('Starting of full feed publication {}, {}'.format(len(feed_str), task), extra={'size': len(feed_str), 'task': task})
# http://docs.celeryproject.org/projects/kombu/en/latest/userguide/producers.html#bypassing-routing-by-using-the-anon-exchange
self.producer.publish(feed_str,
routing_key=task.load_realtime.queue_name,
retry=True,
retry_policy={
'interval_start': 0, # First retry immediately,
'interval_step': 2, # then increase by 2s for every retry.
'interval_max': 10, # but don't exceed 10s between retries.
'max_retries': self.max_retries, # give up after 10 (by default) tries.
})
duration = (datetime.utcnow() - start_datetime).total_seconds()
log.info('End of full feed publication', extra={'duration': duration, 'task': task})
record_call('Full feed publication', size=len(feed_str), routing_key=task.load_realtime.queue_name,
duration=duration, trip_update_count=len(feed.entity),
contributor=task.load_realtime.contributors)
finally:
db.session.remove()
def handle(real_time_update, trip_updates, contributor, is_new_complete=False):
"""
receive a RealTimeUpdate with at least one TripUpdate filled with the data received
by the connector. each TripUpdate is associated with the VehicleJourney returned by jormungandr
Returns real_time_update and the log_dict
"""
if not real_time_update:
raise TypeError()
id_timestamp_tuples = [(tu.vj.navitia_trip_id, tu.vj.get_start_timestamp())
for tu in trip_updates]
old_trip_updates = TripUpdate.find_by_dated_vjs(id_timestamp_tuples)
for trip_update in trip_updates:
# find if there is already a row in db
old = next(
(tu for tu in old_trip_updates
if tu.vj.navitia_trip_id == trip_update.vj.navitia_trip_id and tu.
vj.get_start_timestamp() == trip_update.vj.get_start_timestamp()),
None)
# merge the base schedule, the current realtime, and the new realtime
current_trip_update = merge(trip_update.vj.navitia_vj,
old,
trip_update,
is_new_complete=is_new_complete)
# manage and adjust consistency if possible
if current_trip_update and manage_consistency(current_trip_update):
# we have to link the current_vj_update with the new real_time_update
# this link is done quite late to avoid too soon persistence of trip_update by sqlalchemy
current_trip_update.real_time_updates.append(real_time_update)
persist(real_time_update)
feed = convert_to_gtfsrt(real_time_update.trip_updates)
feed_str = feed.SerializeToString()
publish(feed_str, contributor)
data_time = datetime.datetime.utcfromtimestamp(feed.header.timestamp)
log_dict = {
'contributor': contributor,
'timestamp': data_time,
'trip_update_count': len(feed.entity),
'size': len(feed_str)
}
# After merging trip_updates information of connector realtime, navitia and kirin database, if there is no new
# information destined to navitia, update real_time_update with status = 'KO' and a proper error message.
if not real_time_update.trip_updates and real_time_update.status == 'OK':
real_time_update.status = 'KO'
real_time_update.error = 'No new information destinated to navitia for this {}'.format(
real_time_update.connector)
logging.getLogger(__name__).error('RealTimeUpdate id={}: {}'.format(
real_time_update.id, real_time_update.error))
model.db.session.add(real_time_update)
model.db.session.commit()
return real_time_update, log_dict
def test_populate_pb_with_one_stop_time():
"""
an easy one: we have one vj with only one stop time updated
fill protobuf from trip_update
Verify protobuf
"""
navitia_vj = {
'trip': {
'id': 'vehicle_journey:1'
},
'stop_times': [{
'arrival_time': None,
'departure_time': datetime.time(8, 10),
'stop_point': {
'id': 'sa:1'
}
}, {
'arrival_time': datetime.time(9, 10),
'departure_time': None,
'stop_point': {
'id': 'sa:2'
}
}]
}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8))
trip_update.vj = vj
st = StopTimeUpdate({'id': 'sa:1'},
departure=_dt("8:15"),
arrival=None)
real_time_update = RealTimeUpdate(raw_data=None, connector='ire')
real_time_update.trip_updates.append(trip_update)
trip_update.stop_time_updates.append(st)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version == '1'
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED
pb_stop_time = feed_entity.entity[0].trip_update.stop_time_update[0]
assert pb_stop_time.arrival.time == 0
assert pb_stop_time.departure.time == to_posix_time(_dt("8:15"))
assert pb_stop_time.stop_id == 'sa:1'
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == False
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == False
def callback(body, message):
try:
task = task_pb2.Task()
try:
# `body` is of unicode type, but we need str type for
# `ParseFromString()` to work. It seems to work.
# Maybe kombu estimate that, without any information,
# the body should be something as json, and thus a
# unicode string. On the c++ side, I didn't manage to
# find a way to give a content-type or something like
# that.
body = str(body)
task.ParseFromString(body)
except DecodeError as e:
log.warn('invalid protobuf: {}'.format(str(e)))
return
log.info('getting a request: {}'.format(task))
if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime:
return
begin_date = None
end_date = None
if hasattr(task.load_realtime, "begin_date"):
if task.load_realtime.begin_date:
begin_date = str_to_date(task.load_realtime.begin_date)
if hasattr(task.load_realtime, "end_date"):
if task.load_realtime.end_date:
end_date = str_to_date(task.load_realtime.end_date)
feed = convert_to_gtfsrt(TripUpdate.find_by_contributor_period(task.load_realtime.contributors,
begin_date,
end_date),
gtfs_realtime_pb2.FeedHeader.FULL_DATASET)
with self._get_producer() as producer:
log.info('Publishing full feed...')
producer.publish(feed.SerializeToString(), routing_key=task.load_realtime.queue_name)
log.info('Full feed published.')
finally:
db.session.remove()
def callback(body, message):
task = task_pb2.Task()
try:
# `body` is of unicode type, but we need str type for
# `ParseFromString()` to work. It seems to work.
# Maybe kombu estimate that, without any information,
# the body should be something as json, and thus a
# unicode string. On the c++ side, I didn't manage to
# find a way to give a content-type or something like
# that.
body = str(body)
task.ParseFromString(body)
except DecodeError as e:
log.warn('invalid protobuf: {}'.format(str(e)))
return
log.info('getting a request: {}'.format(task))
if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime:
return
feed = convert_to_gtfsrt(RealTimeUpdate.all(task.load_realtime.contributors))
with self._get_producer() as producer:
producer.publish(feed.SerializeToString(), routing_key=task.load_realtime.queue_name)
def test_populate_pb_with_one_stop_time():
"""
an easy one: we have one vj with only one stop time updated
fill protobuf from trip_update
Verify protobuf
"""
navitia_vj = {'id': 'vehicle_journey:1', 'stop_times': [
{'arrival_time': None, 'departure_time': datetime.time(8, 10), 'stop_point': {'id': 'sa:1'}},
{'arrival_time': datetime.time(9, 10), 'departure_time': None, 'stop_point': {'id': 'sa:2'}}
]}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8))
trip_update.vj = vj
st = StopTimeUpdate({'id': 'sa:1'}, departure=_dt("8:15"), arrival=None)
real_time_update = RealTimeUpdate(raw_data=None, connector='ire')
real_time_update.trip_updates.append(trip_update)
trip_update.stop_time_updates.append(st)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version == '1'
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED
pb_stop_time = feed_entity.entity[0].trip_update.stop_time_update[0]
assert pb_stop_time.arrival.time == 0
assert pb_stop_time.departure.time == to_posix_time(_dt("8:15"))
assert pb_stop_time.stop_id == 'sa:1'
def test_populate_pb_with_cancelation():
"""
VJ cancelation
"""
navitia_vj = {'trip': {'id': 'vehicle_journey:1'}}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8))
trip_update.vj = vj
trip_update.status = 'delete'
trip_update.message = 'Message Test'
real_time_update = RealTimeUpdate(raw_data=None, connector='ire')
trip_update.contributor = 'kisio-digital'
real_time_update.trip_updates.append(trip_update)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version == '1'
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) == True
assert pb_trip_update.Extensions[
kirin_pb2.trip_message] == 'Message Test'
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.CANCELED
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) == True
assert pb_trip_update.trip.Extensions[
kirin_pb2.contributor] == 'kisio-digital'
assert len(feed_entity.entity[0].trip_update.stop_time_update) == 0
def test_populate_pb_with_deleted_stop_time():
"""
test protobuf for partial delete
nav vj
sa:1 * ---- * sa:2 * ---- * sa:3 * ---- * sa:4
we stop the vj from going to sa:3 and sa:4
sa:1 * ---- * sa:2
And we delay sa:1 of 5 minutes
Note: in the message sent sa:2 departure will be removed as well
as sa:3 departures/arrival and sa:4' arrival
"""
#we add another impacted stop time to the Model
from datetime import time
navitia_vj = {
'trip': {
'id': 'vehicle_journey:1'
},
'stop_times': [{
'arrival_time': None,
'departure_time': time(8, 11),
'stop_point': {
'id': 'sa:1',
'stop_area': {
'timezone': 'Europe/Paris'
}
}
}, {
'arrival_time': time(9, 10),
'departure_time': time(9, 11),
'stop_point': {
'id': 'sa:2',
'stop_area': {
'timezone': 'Europe/Paris'
}
}
}, {
'arrival_time': time(10, 10),
'departure_time': time(10, 11),
'stop_point': {
'id': 'sa:3',
'stop_area': {
'timezone': 'Europe/Paris'
}
}
}, {
'arrival_time': time(11, 10),
'departure_time': None,
'stop_point': {
'id': 'sa:4',
'stop_area': {
'timezone': 'Europe/Paris'
}
}
}]
}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8))
trip_update.vj = vj
real_time_update = RealTimeUpdate(raw_data=None,
connector='ire',
contributor='realtime.ire')
real_time_update.trip_updates.append(trip_update)
st = StopTimeUpdate({'id': 'sa:1'},
departure=_dt("8:15"),
departure_delay=timedelta(minutes=5),
arrival=None)
trip_update.stop_time_updates.append(st)
st = StopTimeUpdate(
{'id': 'sa:2'},
arrival=_dt("9:10"),
departure=_dt("9:11"),
dep_status='delete', # we delete the departure
message="bob's on the track")
real_time_update.trip_updates[0].stop_time_updates.append(st)
st = StopTimeUpdate(
{'id': 'sa:3'},
# Note: we still send the departure/arrival for coherence
arrival=_dt("10:10"),
dep_status='delete', # we delete both departure and arrival
departure=_dt("10:11"),
arr_status='delete',
message="bob's on the track")
real_time_update.trip_updates[0].stop_time_updates.append(st)
st = StopTimeUpdate(
{'id': 'sa:4'},
arrival=_dt("11:10"),
arr_status='delete', # we delete only the arrival
departure=_dt("11:11"),
message="bob's on the track")
real_time_update.trip_updates[0].stop_time_updates.append(st)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version, '1'
assert len(feed_entity.entity) == 1
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is False
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED
assert len(pb_trip_update.stop_time_update) == 4
pb_stop_time = pb_trip_update.stop_time_update[0]
assert pb_stop_time.stop_id == 'sa:1'
assert pb_stop_time.arrival.time == 0
assert pb_stop_time.arrival.delay == 0
assert pb_stop_time.departure.time == to_posix_time(_dt("8:15"))
assert pb_stop_time.departure.delay == 5 * 60
assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == ""
pb_stop_time = pb_trip_update.stop_time_update[1]
assert pb_stop_time.stop_id == 'sa:2'
assert pb_stop_time.arrival.time == to_posix_time(_dt("9:10"))
assert pb_stop_time.arrival.delay == 0
# the arrival at the stop is still scheduled, but the departure is skipped
assert pb_stop_time.arrival.Extensions[kirin_pb2.stop_time_event_relationship] == \
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SCHEDULED
assert pb_stop_time.departure.time == to_posix_time(_dt("9:11"))
assert pb_stop_time.departure.delay == 0
assert pb_stop_time.departure.Extensions[kirin_pb2.stop_time_event_relationship] == \
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED
assert pb_stop_time.Extensions[
kirin_pb2.stoptime_message] == "bob's on the track"
pb_stop_time = pb_trip_update.stop_time_update[2]
assert pb_stop_time.stop_id == 'sa:3'
# both departure and arrival are SKIPPED
assert pb_stop_time.arrival.time == to_posix_time(_dt("10:10"))
assert pb_stop_time.arrival.delay == 0
assert pb_stop_time.arrival.Extensions[kirin_pb2.stop_time_event_relationship] == \
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED
assert pb_stop_time.departure.time == to_posix_time(_dt("10:11"))
assert pb_stop_time.departure.delay == 0
assert pb_stop_time.departure.Extensions[kirin_pb2.stop_time_event_relationship] == \
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED
assert pb_stop_time.Extensions[
kirin_pb2.stoptime_message] == "bob's on the track"
pb_stop_time = pb_trip_update.stop_time_update[3]
assert pb_stop_time.stop_id == 'sa:4'
# only the arrival is skipped, the departure does not exists as it's the last stop
assert pb_stop_time.arrival.time == to_posix_time(_dt("11:10"))
assert pb_stop_time.arrival.delay == 0
assert pb_stop_time.arrival.Extensions[kirin_pb2.stop_time_event_relationship] == \
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED
assert pb_stop_time.departure.time == to_posix_time(_dt("11:11"))
assert pb_stop_time.departure.delay == 0
assert pb_stop_time.departure.Extensions[kirin_pb2.stop_time_event_relationship] == \
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SCHEDULED
assert pb_stop_time.Extensions[
kirin_pb2.stoptime_message] == "bob's on the track"
def test_populate_pb_with_two_stop_time():
"""
an easy one: we have one vj with only one stop time updated
fill protobuf from trip_update
Verify protobuf
"""
#we add another impacted stop time to the Model
navitia_vj = {
'trip': {
'id': 'vehicle_journey:1'
},
'stop_times': [{
'arrival_time': None,
'departure_time': datetime.time(8, 10),
'stop_point': {
'id': 'sa:1',
'stop_area': {
'timezone': 'Europe/Paris'
}
}
}, {
'arrival_time': datetime.time(9, 10),
'departure_time': None,
'stop_point': {
'id': 'sa:2',
'stop_area': {
'timezone': 'Europe/Paris'
}
}
}]
}
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(navitia_vj, datetime.date(2015, 9, 8))
trip_update.vj = vj
real_time_update = RealTimeUpdate(raw_data=None,
connector='ire',
contributor='realtime.ire')
real_time_update.trip_updates.append(trip_update)
st = StopTimeUpdate({'id': 'sa:1'},
departure=_dt("8:15"),
departure_delay=timedelta(minutes=5),
arrival=None)
trip_update.stop_time_updates.append(st)
st = StopTimeUpdate({'id': 'sa:2'},
departure=_dt("8:21"),
departure_delay=timedelta(minutes=-40),
arrival=_dt("8:20"),
arrival_delay=timedelta(minutes=-40),
message="bob's on the track")
real_time_update.trip_updates[0].stop_time_updates.append(st)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version, '1'
assert len(feed_entity.entity) == 1
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is False
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED
assert len(pb_trip_update.stop_time_update) == 2
pb_stop_time = pb_trip_update.stop_time_update[0]
assert pb_stop_time.stop_id == 'sa:1'
assert pb_stop_time.arrival.time == 0
assert pb_stop_time.arrival.delay == 0
assert pb_stop_time.departure.time == to_posix_time(_dt("8:15"))
assert pb_stop_time.departure.delay == 5 * 60
assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == ""
pb_stop_time = pb_trip_update.stop_time_update[1]
assert pb_stop_time.stop_id == 'sa:2'
assert pb_stop_time.arrival.time == to_posix_time(_dt("8:20"))
assert pb_stop_time.arrival.delay == -40 * 60
assert pb_stop_time.departure.time == to_posix_time(_dt("8:21"))
assert pb_stop_time.departure.delay == -40 * 60
assert pb_stop_time.Extensions[
kirin_pb2.stoptime_message] == "bob's on the track"
def test_populate_pb_with_deleted_stop_time():
"""
test protobuf for partial delete
nav vj
sa:1 * ---- * sa:2 * ---- * sa:3 * ---- * sa:4
we stop the vj from going to sa:3 and sa:4
sa:1 * ---- * sa:2
And we delay sa:1 of 5 minutes
Note: in the message sent sa:2 departure will be removed as well
as sa:3 departures/arrival and sa:4' arrival
"""
# we add another impacted stop time to the Model
from datetime import time
navitia_vj = {
"trip": {
"id": "vehicle_journey:1"
},
"stop_times": [
{
"utc_arrival_time": None,
"utc_departure_time": time(6, 11),
"stop_point": {
"id": "sa:1",
"stop_area": {
"timezone": "Europe/Paris"
}
},
},
{
"utc_arrival_time": time(7, 10),
"utc_departure_time": time(7, 11),
"stop_point": {
"id": "sa:2",
"stop_area": {
"timezone": "Europe/Paris"
}
},
},
{
"utc_arrival_time": time(8, 10),
"utc_departure_time": time(8, 11),
"stop_point": {
"id": "sa:3",
"stop_area": {
"timezone": "Europe/Paris"
}
},
},
{
"utc_arrival_time": time(9, 10),
"utc_departure_time": None,
"stop_point": {
"id": "sa:4",
"stop_area": {
"timezone": "Europe/Paris"
}
},
},
],
}
with app.app_context():
vj = VehicleJourney(navitia_vj,
datetime.datetime(2015, 9, 8, 5, 11, 0),
datetime.datetime(2015, 9, 8, 10, 10, 0))
trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update.trip_updates.append(trip_update)
st = StopTimeUpdate({"id": "sa:1"},
departure=_dt("8:15"),
departure_delay=timedelta(minutes=5),
arrival=None)
trip_update.stop_time_updates.append(st)
st = StopTimeUpdate(
{"id": "sa:2"},
arrival=_dt("9:10"),
departure=_dt("9:11"),
dep_status="delete", # we delete the departure
message="bob's on the track",
)
real_time_update.trip_updates[0].stop_time_updates.append(st)
st = StopTimeUpdate(
{"id": "sa:3"},
# Note: we still send the departure/arrival for coherence
arrival=_dt("10:10"),
dep_status="delete", # we delete both departure and arrival
departure=_dt("10:11"),
arr_status="delete",
message="bob's on the track",
)
real_time_update.trip_updates[0].stop_time_updates.append(st)
st = StopTimeUpdate(
{"id": "sa:4"},
arrival=_dt("11:10"),
arr_status="delete", # we delete only the arrival
departure=_dt("11:11"),
message="bob's on the track",
)
real_time_update.trip_updates[0].stop_time_updates.append(st)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version, "1"
assert len(feed_entity.entity) == 1
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == "vehicle_journey:1"
assert pb_trip_update.trip.start_date == "20150908"
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True
assert pb_trip_update.trip.HasExtension(kirin_pb2.company_id) is False
assert pb_trip_update.HasExtension(kirin_pb2.effect) is False
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED
assert len(pb_trip_update.stop_time_update) == 4
pb_stop_time = pb_trip_update.stop_time_update[0]
assert pb_stop_time.stop_id == "sa:1"
assert pb_stop_time.arrival.time == 0
assert pb_stop_time.arrival.delay == 0
assert pb_stop_time.departure.time == to_posix_time(_dt("8:15"))
assert pb_stop_time.departure.delay == 5 * 60
assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == ""
pb_stop_time = pb_trip_update.stop_time_update[1]
assert pb_stop_time.stop_id == "sa:2"
assert pb_stop_time.arrival.time == to_posix_time(_dt("9:10"))
assert pb_stop_time.arrival.delay == 0
# the arrival at the stop is still scheduled, but the departure is skipped
assert (pb_stop_time.arrival.Extensions[
kirin_pb2.stop_time_event_relationship] ==
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SCHEDULED)
assert pb_stop_time.departure.time == to_posix_time(_dt("9:11"))
assert pb_stop_time.departure.delay == 0
assert (pb_stop_time.departure.Extensions[
kirin_pb2.stop_time_event_relationship] ==
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED)
assert pb_stop_time.Extensions[
kirin_pb2.stoptime_message] == "bob's on the track"
pb_stop_time = pb_trip_update.stop_time_update[2]
assert pb_stop_time.stop_id == "sa:3"
# both departure and arrival are SKIPPED
assert pb_stop_time.arrival.time == to_posix_time(_dt("10:10"))
assert pb_stop_time.arrival.delay == 0
assert (pb_stop_time.arrival.Extensions[
kirin_pb2.stop_time_event_relationship] ==
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED)
assert pb_stop_time.departure.time == to_posix_time(_dt("10:11"))
assert pb_stop_time.departure.delay == 0
assert (pb_stop_time.departure.Extensions[
kirin_pb2.stop_time_event_relationship] ==
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED)
assert pb_stop_time.Extensions[
kirin_pb2.stoptime_message] == "bob's on the track"
pb_stop_time = pb_trip_update.stop_time_update[3]
assert pb_stop_time.stop_id == "sa:4"
# only the arrival is skipped, the departure does not exists as it's the last stop
assert pb_stop_time.arrival.time == to_posix_time(_dt("11:10"))
assert pb_stop_time.arrival.delay == 0
assert (pb_stop_time.arrival.Extensions[
kirin_pb2.stop_time_event_relationship] ==
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SKIPPED)
assert pb_stop_time.departure.time == to_posix_time(_dt("11:11"))
assert pb_stop_time.departure.delay == 0
assert (pb_stop_time.departure.Extensions[
kirin_pb2.stop_time_event_relationship] ==
gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.SCHEDULED)
assert pb_stop_time.Extensions[
kirin_pb2.stoptime_message] == "bob's on the track"
def test_populate_pb_with_one_stop_time():
"""
an easy one: we have one vj with only one stop time updated
fill protobuf from trip_update
Verify protobuf
"""
navitia_vj = {
"trip": {
"id": "vehicle_journey:1"
},
"stop_times": [
{
"utc_arrival_time": None,
"utc_departure_time": datetime.time(6, 10),
"stop_point": {
"id": "sa:1",
"stop_area": {
"timezone": "Europe/Paris"
}
},
},
{
"utc_arrival_time": datetime.time(7, 10),
"utc_departure_time": None,
"stop_point": {
"id": "sa:2",
"stop_area": {
"timezone": "Europe/Paris"
}
},
},
],
}
with app.app_context():
vj = VehicleJourney(navitia_vj,
datetime.datetime(2015, 9, 8, 5, 10, 0),
datetime.datetime(2015, 9, 8, 8, 10, 0))
trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID)
st = StopTimeUpdate({"id": "sa:1"},
departure=_dt("8:15"),
arrival=None)
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update.trip_updates.append(trip_update)
trip_update.stop_time_updates.append(st)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version == "1"
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == "vehicle_journey:1"
assert pb_trip_update.trip.start_date == "20150908"
assert pb_trip_update.trip.schedule_relationship == gtfs_realtime_pb2.TripDescriptor.SCHEDULED
pb_stop_time = feed_entity.entity[0].trip_update.stop_time_update[0]
assert pb_stop_time.arrival.time == 0
assert pb_stop_time.departure.time == to_posix_time(_dt("8:15"))
assert pb_stop_time.stop_id == "sa:1"
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True
assert pb_trip_update.trip.HasExtension(kirin_pb2.company_id) is False
assert pb_trip_update.HasExtension(kirin_pb2.effect) is False
def test_populate_pb_for_added_trip():
"""
For an added trip we don't call navitia to get a vj instead we create and initialize one vj only with 'id'
Fill protobuf from trip_update
Verify protobuf
"""
navitia_vj = make_navitia_empty_vj('vehicle_journey:1')
with app.app_context():
trip_update = TripUpdate()
vj = VehicleJourney(
navitia_vj,
utc_since_dt=utc.localize(datetime.datetime(2015, 9, 8, 5, 10, 0)),
utc_until_dt=utc.localize(datetime.datetime(2015, 9, 8, 8, 10, 0)),
vj_start_dt=utc.localize(datetime.datetime(2015, 9, 8, 5, 10, 0)))
trip_update.vj = vj
trip_update.status = 'add'
trip_update.effect = 'ADDITIONAL_SERVICE'
trip_update.physical_mode_id = 'physical_mode:LongDistanceTrain'
real_time_update = RealTimeUpdate(raw_data=None,
connector='cots',
contributor='realtime.cots')
real_time_update.trip_updates.append(trip_update)
st = StopTimeUpdate({'id': 'sa:1'},
departure=_dt("8:15"),
departure_delay=timedelta(minutes=5),
arrival=None,
arr_status='none',
dep_status='add')
trip_update.stop_time_updates.append(st)
st = StopTimeUpdate({'id': 'sa:2'},
departure=_dt("8:21"),
departure_delay=timedelta(minutes=-40),
arrival=_dt("8:20"),
arrival_delay=timedelta(minutes=-40),
message="bob's on the track",
arr_status='add',
dep_status='none')
trip_update.stop_time_updates.append(st)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version, '1'
assert len(feed_entity.entity) == 1
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == 'OCE:SN:vehicle_journey:1'
assert pb_trip_update.trip.start_date == '20150908'
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is False
assert pb_trip_update.trip.HasExtension(kirin_pb2.company_id) is False
assert pb_trip_update.HasExtension(kirin_pb2.effect) is True
assert pb_trip_update.Extensions[
kirin_pb2.effect] == gtfs_realtime_pb2.Alert.ADDITIONAL_SERVICE
assert pb_trip_update.vehicle.Extensions[
kirin_pb2.physical_mode_id] == 'physical_mode:LongDistanceTrain'
assert len(pb_trip_update.stop_time_update) == 2
pb_stop_time = pb_trip_update.stop_time_update[0]
assert pb_stop_time.stop_id == 'sa:1'
assert pb_stop_time.arrival.time == 0
assert pb_stop_time.arrival.delay == 0
assert pb_stop_time.departure.time == to_posix_time(_dt("8:15"))
assert pb_stop_time.departure.delay == 5 * 60
assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == ""
assert pb_stop_time.arrival.Extensions[
kirin_pb2.stop_time_event_status] == kirin_pb2.SCHEDULED
assert pb_stop_time.departure.Extensions[
kirin_pb2.stop_time_event_status] == kirin_pb2.ADDED
pb_stop_time = pb_trip_update.stop_time_update[1]
assert pb_stop_time.stop_id == 'sa:2'
assert pb_stop_time.arrival.time == to_posix_time(_dt("8:20"))
assert pb_stop_time.arrival.delay == -40 * 60
assert pb_stop_time.departure.time == to_posix_time(_dt("8:21"))
assert pb_stop_time.departure.delay == -40 * 60
assert pb_stop_time.Extensions[
kirin_pb2.stoptime_message] == "bob's on the track"
assert pb_stop_time.arrival.Extensions[
kirin_pb2.stop_time_event_status] == kirin_pb2.ADDED
assert pb_stop_time.departure.Extensions[
kirin_pb2.stop_time_event_status] == kirin_pb2.SCHEDULED
def test_populate_pb_for_added_trip():
"""
For an added trip we don't call navitia to get a vj instead we create and initialize one vj only with 'id'
Fill protobuf from trip_update
Verify protobuf
"""
navitia_vj = make_navitia_empty_vj("vehicle_journey:1")
with app.app_context():
vj = VehicleJourney(
navitia_vj,
since_dt=datetime.datetime(2015, 9, 8, 5, 10, 0),
until_dt=datetime.datetime(2015, 9, 8, 8, 10, 0),
vj_start_dt=datetime.datetime(2015, 9, 8, 5, 10, 0),
)
trip_update = TripUpdate(vj=vj, contributor_id=COTS_CONTRIBUTOR_ID)
trip_update.vj = vj
trip_update.status = "add"
trip_update.effect = "ADDITIONAL_SERVICE"
trip_update.physical_mode_id = "physical_mode:LongDistanceTrain"
real_time_update = make_rt_update(
raw_data=None,
connector_type=ConnectorType.cots.value,
contributor_id=COTS_CONTRIBUTOR_ID)
real_time_update.trip_updates.append(trip_update)
st = StopTimeUpdate(
{"id": "sa:1"},
departure=_dt("8:15"),
departure_delay=timedelta(minutes=5),
arrival=None,
arr_status="none",
dep_status="add",
)
trip_update.stop_time_updates.append(st)
st = StopTimeUpdate(
{"id": "sa:2"},
departure=_dt("8:21"),
departure_delay=timedelta(minutes=-40),
arrival=_dt("8:20"),
arrival_delay=timedelta(minutes=-40),
message="bob's on the track",
arr_status="add",
dep_status="none",
)
trip_update.stop_time_updates.append(st)
db.session.add(real_time_update)
db.session.commit()
feed_entity = convert_to_gtfsrt(real_time_update.trip_updates)
assert feed_entity.header.incrementality == gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
assert feed_entity.header.gtfs_realtime_version, "1"
assert len(feed_entity.entity) == 1
pb_trip_update = feed_entity.entity[0].trip_update
assert pb_trip_update.trip.trip_id == "OCE:SN:vehicle_journey:1"
assert pb_trip_update.trip.start_date == "20150908"
assert pb_trip_update.HasExtension(kirin_pb2.trip_message) is False
assert pb_trip_update.trip.HasExtension(kirin_pb2.contributor) is True
assert pb_trip_update.trip.HasExtension(kirin_pb2.company_id) is False
assert pb_trip_update.HasExtension(kirin_pb2.effect) is True
assert pb_trip_update.Extensions[
kirin_pb2.effect] == gtfs_realtime_pb2.Alert.ADDITIONAL_SERVICE
assert pb_trip_update.vehicle.Extensions[
kirin_pb2.physical_mode_id] == "physical_mode:LongDistanceTrain"
assert len(pb_trip_update.stop_time_update) == 2
pb_stop_time = pb_trip_update.stop_time_update[0]
assert pb_stop_time.stop_id == "sa:1"
assert pb_stop_time.arrival.time == 0
assert pb_stop_time.arrival.delay == 0
assert pb_stop_time.departure.time == to_posix_time(_dt("8:15"))
assert pb_stop_time.departure.delay == 5 * 60
assert pb_stop_time.Extensions[kirin_pb2.stoptime_message] == ""
assert pb_stop_time.arrival.Extensions[
kirin_pb2.stop_time_event_status] == kirin_pb2.SCHEDULED
assert pb_stop_time.departure.Extensions[
kirin_pb2.stop_time_event_status] == kirin_pb2.ADDED
pb_stop_time = pb_trip_update.stop_time_update[1]
assert pb_stop_time.stop_id == "sa:2"
assert pb_stop_time.arrival.time == to_posix_time(_dt("8:20"))
assert pb_stop_time.arrival.delay == -40 * 60
assert pb_stop_time.departure.time == to_posix_time(_dt("8:21"))
assert pb_stop_time.departure.delay == -40 * 60
assert pb_stop_time.Extensions[
kirin_pb2.stoptime_message] == "bob's on the track"
assert pb_stop_time.arrival.Extensions[
kirin_pb2.stop_time_event_status] == kirin_pb2.ADDED
assert pb_stop_time.departure.Extensions[
kirin_pb2.stop_time_event_status] == kirin_pb2.SCHEDULED
def _on_request(self, message):
log = logging.getLogger(__name__)
status = "OK"
log_dict = {}
start_datetime = datetime.utcnow()
try:
task = task_pb2.Task()
try:
# `body` is a string, but we need binary type for
# `ParseFromString()` to work. It seems to work.
body = bytes(message.payload, encoding="utf-8")
task.ParseFromString(body)
except DecodeError as e:
log.warning("invalid protobuf: {}".format(str(e)))
return
log.info("Getting a full feed publication request",
extra={"task": task})
if task.action != task_pb2.LOAD_REALTIME or not task.load_realtime:
return
begin_date = None
end_date = None
if hasattr(task.load_realtime,
"begin_date") and task.load_realtime.begin_date:
begin_date = str_to_date(task.load_realtime.begin_date)
if hasattr(task.load_realtime,
"end_date") and task.load_realtime.end_date:
end_date = str_to_date(task.load_realtime.end_date)
feed = convert_to_gtfsrt(
TripUpdate.find_by_contributor_period(
task.load_realtime.contributors, begin_date, end_date),
gtfs_realtime_pb2.FeedHeader.FULL_DATASET,
)
feed_str = feed.SerializeToString()
log_dict.update({
"contributors": task.load_realtime.contributors,
"routing_key": task.load_realtime.queue_name,
"output_trip_update_count": len(feed.entity),
"output_feed_size": sys.getsizeof(feed_str),
})
log.info(
"Starting of full feed publication {}, {}".format(
len(feed_str), task),
extra={
"size": len(feed_str),
"task": task
},
)
record_custom_parameter("contributors",
task.load_realtime.contributors)
# http://docs.celeryproject.org/projects/kombu/en/latest/userguide/producers.html#bypassing-routing-by-using-the-anon-exchange
self.producer.publish(
feed_str,
routing_key=task.load_realtime.queue_name,
retry=True,
retry_policy={
"interval_start": 0, # First retry immediately,
"interval_step": 2, # then increase by 2s for every retry.
"interval_max":
10, # but don't exceed 10s between retries.
"max_retries":
self.max_retries, # give up after 10 (by default) tries.
},
)
except Exception as e:
status = "failure"
log_dict.update({"reason": str(e)})
record_custom_parameter("reason", str(e))
finally:
duration = (datetime.utcnow() - start_datetime).total_seconds()
log_dict.update({"duration": duration})
record_call("kirin_reload_info", status, **log_dict)
log.info("End of full feed publication",
extra={
"duration": duration,
"task": task
})
db.session.remove()
return log_dict
