def setup_database():
"""
we create two realtime_updates with the same vj but for different date
and return a vj for navitia
"""
with app.app_context():
vj1 = model.VehicleJourney({'trip': {'id': 'vj:1'}}, date(2015, 11, 4))
vj2 = model.VehicleJourney({'trip': {'id': 'vj:2'}}, date(2015, 11, 4))
vj3 = model.VehicleJourney({'trip': {'id': 'vj:3'}}, date(2015, 11, 4))
tu1 = model.TripUpdate(vj1, contributor='realtime.ire')
tu2 = model.TripUpdate(vj2, contributor='realtime.ire')
tu3 = model.TripUpdate(vj3, contributor='realtime.timeo')
rtu1 = model.RealTimeUpdate(None, 'ire')
rtu1.created_at = datetime(2015, 11, 4, 6, 32)
rtu1.trip_updates.append(tu1)
model.db.session.add(rtu1)
rtu2 = model.RealTimeUpdate(None, 'ire')
rtu2.created_at = datetime(2015, 11, 4, 7, 32)
rtu2.trip_updates.append(tu2)
model.db.session.add(rtu2)
rtu3 = model.RealTimeUpdate(None, 'ire')
rtu3.created_at = datetime(2015, 11, 4, 7, 42)
rtu3.trip_updates.append(tu3)
model.db.session.add(rtu3)
model.db.session.commit()
def _build_one_trip_update(self, gtfsrt_trip_update, gtfsrt_data_time):
"""
Build the smallest TripUpdate from GTFSRT trip, don't try to fill missing information yet
Goal is to just convert information to Kirin's model
(and add navitia PT objects information, like navitia ids, etc.)
"""
if not gtfsrt_trip_update.HasField("trip") or not gtfsrt_trip_update.trip.HasField("trip_id"):
# TODO: log and count in NR
return None
# Processing only scheduled trips currently
if (
gtfsrt_trip_update.trip.HasField("schedule_relationship")
and gtfsrt_trip_update.trip.schedule_relationship != gtfs_realtime_pb2.TripDescriptor.SCHEDULED
):
# TODO: log and count in NR
return None
since_dt = floor_datetime(gtfsrt_data_time - self.period_filter_tolerance)
until_dt = floor_datetime(gtfsrt_data_time + self.period_filter_tolerance + datetime.timedelta(hours=1))
self.log.info("get_nav_vj")
vj = self._get_navitia_vj(gtfsrt_trip_update.trip.trip_id, since_dt=since_dt, until_dt=until_dt)
if vj is None: # if not able to find the base-schedule trip, reject GTFSRT trip as no ADD is handled
# TODO: log and count in NR
return None
trip_update = model.TripUpdate(vj=vj, contributor_id=self.contributor.id)
if gtfsrt_trip_update.trip.HasField("schedule_relationship"):
trip_effect = trip_gtfsrt_status_to_effect.get(gtfsrt_trip_update.trip.schedule_relationship)
if trip_effect is not None:
trip_update.effect = trip_effect.name
trip_update.status = get_trip_status_from_effect(trip_update.effect)
# this variable is used to store the last stop_time's departure in order to check the stop_time consistency
# ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure
previous_rt_stop_event_time = timestamp_to_utc_naive_dt(0)
for gtfsrt_stop in gtfsrt_trip_update.stop_time_update:
try:
st_update, last_stop_event_time = self._build_one_stop_time_update(
gtfsrt_stop=gtfsrt_stop,
nav_vj=vj.navitia_vj,
previous_rt_stop_event_time=previous_rt_stop_event_time,
)
if None not in [st_update, last_stop_event_time]:
trip_update.stop_time_updates.append(st_update)
previous_rt_stop_event_time = last_stop_event_time
except Exception as _:
del trip_update.stop_time_updates[:]
raise
return trip_update
def _make_trip_update(self, vj, xml_modification):
"""
create the TripUpdate object
"""
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
delay = xml_modification.find('HoraireProjete')
if delay:
trip_update.status = 'update'
for downstream_point in delay.iter('PointAval'):
# we need only to consider the station
if not as_bool(get_value(downstream_point,
'IndicateurPRGare')):
continue
nav_st = self._get_navitia_stop_time(downstream_point,
vj.navitia_vj)
if nav_st is None:
continue
nav_stop = nav_st.get('stop_point', {})
dep_delay, dep_status = self._get_delay(
downstream_point.find('TypeHoraire/Depart'))
arr_delay, arr_status = self._get_delay(
downstream_point.find('TypeHoraire/Arrivee'))
message = get_value(downstream_point, 'MotifExterne')
st_update = model.StopTimeUpdate(nav_stop,
departure_delay=dep_delay,
arrival_delay=arr_delay,
dep_status=dep_status,
arr_status=arr_status,
message=message)
trip_update.stop_time_updates.append(st_update)
removal = xml_modification.find('Suppression')
if removal:
if get_value(removal, 'TypeSuppression') == 'T':
trip_update.status = 'delete'
trip_update.stop_time_updates = []
elif get_value(removal, 'TypeSuppression') == 'P':
trip_update.status = 'update'
xml_prdebut = removal.find('PRDebut')
if xml_prdebut:
trip_update.message = get_value(xml_prdebut, 'MotifExterne')
return trip_update
def _make_trip_updates(self, input_trip_update, data_time):
vjs = self._get_navitia_vjs(input_trip_update.trip, data_time=data_time)
trip_updates = []
for vj in vjs:
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
trip_updates.append(trip_update)
for input_st_update in input_trip_update.stop_time_update:
st_update = self._make_stoptime_update(input_st_update, vj.navitia_vj)
if not st_update:
continue
trip_update.stop_time_updates.append(st_update)
return trip_updates
def build_trip_update_if_modified(
db_trip_update: Optional[model.TripUpdate],
old_stus: List[model.StopTimeUpdate],
new_trip_update: model.TripUpdate,
new_stus: List[model.StopTimeUpdate],
contributor_id: str,
) -> Optional[model.TripUpdate]:
"""
Build and return a consolidated TripUpdate (with its attached StopTimeUpdates) if
modifications appeared in new one
:param db_trip_update: TripUpdate obtained from db if previous realtime processing result was stored (or None)
:param old_stus: ordered STU list of previous realtime processing result (or corresponding list from base-schedule)
:param new_trip_update: TripUpdate from new feed being processed
:param new_stus: ordered STU list of new feed being processed
:param contributor_id: contributor's id
:return: TripUpdate with attached StopTimeUpdates if new is different from old (or from base-schedule),
otherwise None
"""
old_tu = db_trip_update or model.TripUpdate(vj=None,
contributor_id=contributor_id)
if are_trip_updates_equal(
left_tu=old_tu,
left_stus=old_stus,
right_tu=new_trip_update,
right_stus=new_stus,
):
return None
# Update existing TripUpdate in order to:
# * avoid duplicates (unicity constraint)
# * keep created_at
# * preserve performance (ORM objects are heavy)
res = db_trip_update if db_trip_update else new_trip_update
res.copy_processed_attributes(new_trip_update)
# As SQLAlchemy persists TU in db when STU are attached,
# attach StopTimeUpdates only at the end to avoid breaking unicity constraint that could happen
# when new_trip_update exists at the same time than the result of a previous realtime processing
res.stop_time_updates = new_stus
return res
def _make_trip_update(self, vj, xml_modification):
"""
create the TripUpdate object
"""
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
delay = xml_modification.find('HoraireProjete')
if delay:
trip_update.status = 'update'
for downstream_point in delay.iter('PointAval'):
# we need only to consider the station
if not as_bool(get_value(downstream_point,
'IndicateurPRGare')):
continue
nav_st, log_dict = self._get_navitia_stop_time(
downstream_point, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict['log'],
contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology',
extra=log_dict)
if nav_st is None:
continue
nav_stop = nav_st.get('stop_point', {})
dep_delay, dep_status = self._get_delay(
downstream_point.find('TypeHoraire/Depart'))
arr_delay, arr_status = self._get_delay(
downstream_point.find('TypeHoraire/Arrivee'))
message = get_value(downstream_point,
'MotifExterne',
nullabe=True)
st_update = model.StopTimeUpdate(nav_stop,
departure_delay=dep_delay,
arrival_delay=arr_delay,
dep_status=dep_status,
arr_status=arr_status,
message=message)
trip_update.stop_time_updates.append(st_update)
removal = xml_modification.find('Suppression')
if removal:
xml_prdebut = removal.find('PRDebut')
if get_value(removal, 'TypeSuppression') == 'T':
trip_update.status = 'delete'
trip_update.stop_time_updates = []
elif get_value(removal, 'TypeSuppression') == 'P':
# it's a partial delete
trip_update.status = 'update'
deleted_points = itertools.chain([removal.find('PRDebut')],
removal.iter('PointSupprime'),
[removal.find('PRFin')])
for deleted_point in deleted_points:
# we need only to consider the stations
if not as_bool(get_value(deleted_point,
'IndicateurPRGare')):
continue
nav_st, log_dict = self._get_navitia_stop_time(
deleted_point, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict['log'],
contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology',
extra=log_dict)
if nav_st is None:
continue
nav_stop = nav_st.get('stop_point', {})
# if the <Depart>/<Arrivee> tags are there, the departure/arrival has been deleted
# regardless of the <Etat> tag
dep_deleted = deleted_point.find(
'TypeHoraire/Depart') is not None
arr_deleted = deleted_point.find(
'TypeHoraire/Arrivee') is not None
dep_status = 'delete' if dep_deleted else 'none'
arr_status = 'delete' if arr_deleted else 'none'
message = get_value(deleted_point,
'MotifExterne',
nullabe=True)
st_update = model.StopTimeUpdate(nav_stop,
dep_status=dep_status,
arr_status=arr_status,
message=message)
trip_update.stop_time_updates.append(st_update)
if xml_prdebut:
trip_update.message = get_value(xml_prdebut,
'MotifExterne',
nullabe=True)
return trip_update
def _make_trip_updates(self, input_trip_update, input_data_time):
"""
If trip_update.stop_time_updates is not a strict ending subset of vj.stop_times we reject the trip update
On the other hand:
1. For the stop point present in trip_update.stop_time_updates we create a trip_update merging
informations with that of navitia stop
2. For the first stop point absent in trip_update.stop_time_updates we create a stop_time_update
with no delay for that stop
"""
vjs = self._get_navitia_vjs(input_trip_update.trip,
input_data_time=input_data_time)
trip_updates = []
for vj in vjs:
trip_update = model.TripUpdate(vj=vj,
contributor_id=self.contributor.id)
highest_st_status = ModificationType.none.name
is_tu_valid = True
vj_stop_order = len(vj.navitia_vj.get("stop_times", [])) - 1
for vj_stop, tu_stop in itertools.zip_longest(
reversed(vj.navitia_vj.get("stop_times", [])),
reversed(input_trip_update.stop_time_update)):
if vj_stop is None:
is_tu_valid = False
break
vj_stop_point = vj_stop.get("stop_point")
if vj_stop_point is None:
is_tu_valid = False
break
if tu_stop is not None:
if self._get_stop_code(vj_stop_point) != tu_stop.stop_id:
is_tu_valid = False
break
tu_stop.stop_sequence = vj_stop_order
st_update = _make_stoptime_update(tu_stop, vj_stop_point)
if st_update is not None:
trip_update.stop_time_updates.append(st_update)
else:
# Initialize stops absent in trip_updates but present in vj
st_update = _init_stop_update(vj_stop_point, vj_stop_order)
if st_update is not None:
trip_update.stop_time_updates.append(st_update)
for status in [
st_update.departure_status, st_update.arrival_status
]:
highest_st_status = get_higher_status(
highest_st_status, status)
vj_stop_order -= 1
if is_tu_valid:
# Since vj.stop_times are managed in reversed order, we re sort stop_time_updates by order.
trip_update.stop_time_updates.sort(key=lambda x: x.order)
trip_update.effect = get_effect_from_modification_type(
highest_st_status)
trip_updates.append(trip_update)
else:
self.log.warning(
"stop_time_update do not match with stops in navitia for trip : {} timestamp: {}"
.format(input_trip_update.trip.trip_id,
calendar.timegm(input_data_time.utctimetuple())))
record_internal_failure(
"stop_time_update do not match with stops in navitia",
contributor=self.contributor.id)
del trip_update.stop_time_updates[:]
return trip_updates
def _make_trip_update(self, json_train, vj):
trip_update = model.TripUpdate(vj=vj,
contributor_id=self.contributor.id)
trip_update.headsign = json_train.get("numero")
company_id = json_train.get("operateur").get("codeOperateur")
trip_update.company_id = self._get_navitia_company_id(company_id)
physical_mode = json_train.get("modeTransport").get("typeMode")
trip_update.physical_mode_id = self._get_navitia_physical_mode_id(
physical_mode)
trip_status_type = trip_piv_status_to_effect.get(
json_train.get("evenement").get("type"), TripEffect.UNDEFINED)
trip_update.message = json_train.get("evenement").get("texte")
trip_update.effect = trip_status_type.name
if trip_status_type == TripEffect.NO_SERVICE:
# the whole trip is deleted
trip_update.status = ModificationType.delete.name
trip_update.stop_time_updates = []
return trip_update
elif trip_status_type == TripEffect.ADDITIONAL_SERVICE:
trip_update.status = ModificationType.add.name
else:
trip_update.status = ModificationType.update.name
highest_st_status = ModificationType.none.name
ads = get_value(get_value(json_train, "listeArretsDesserte"), "arret")
# this variable is used to store the last stop_time's departure in order to check the stop_time consistency
# ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure
previous_rt_stop_event_time = datetime.datetime.utcfromtimestamp(0)
for arret in ads:
# retrieve navitia's stop_point corresponding to the current PIV ad
nav_stop, log_dict = self._get_navitia_stop_point(
arret, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict["log"],
contributor=self.contributor.id)
log_dict.update({"contributor": self.contributor.id})
logging.getLogger(__name__).info("metrology", extra=log_dict)
if nav_stop is None:
continue # simply skip stop_times at unknown stop areas
st_update = model.StopTimeUpdate(nav_stop)
trip_update.stop_time_updates.append(st_update)
st_update.message = _get_message(arret)
for event_toggle in ["arrivee", "depart"]:
event = get_value(arret, event_toggle, nullable=True)
if event is None:
continue
piv_disruption = event.get("evenement", {})
piv_event_status = event.get("statutModification",
piv_disruption.get("type", None))
if not piv_event_status or piv_event_status in MANAGED_STOP_EVENTS:
piv_event_datetime = get_value(event,
"dateHeureReelle",
nullable=True)
event_datetime = str_to_utc_naive_dt(
piv_event_datetime) if piv_event_datetime else None
if event_datetime:
setattr(st_update,
STOP_EVENT_DATETIME_MAP[event_toggle],
event_datetime)
if piv_event_status in ["SUPPRESSION_PARTIELLE"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.delete.name)
elif piv_event_status in ["SUPPRESSION_DETOURNEMENT"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.deleted_for_detour.name)
elif piv_event_status in ["CREATION"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.add.name)
elif piv_event_status in ["CREATION_DETOURNEMENT"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.added_for_detour.name)
elif trip_status_type == TripEffect.ADDITIONAL_SERVICE:
# In this case we want to create schedules taking into account the delay
# without adding the delay a second time
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.add.name)
elif piv_event_status in [
"RETARD_OBSERVE", "RETARD_PROJETE"
]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.update.name)
if piv_disruption:
piv_event_delay = piv_disruption.get(
"retard", {}).get("duree", 0)
setattr(st_update, DELAY_MAP[event_toggle],
as_duration(piv_event_delay *
60)) # minutes
else:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.none.name)
# otherwise let those be none
else:
raise InvalidArguments(
"invalid value {s} for field {t}/statutModification or {t}/evenement/type"
.format(s=piv_event_status, t=event_toggle))
event_status = getattr(st_update, STATUS_MAP[event_toggle],
ModificationType.none.name)
highest_st_status = get_higher_status(highest_st_status,
event_status)
mdi = (arret[STOP_EVENT_MDI_MAP[event_toggle]] if arret.get(
STOP_EVENT_MDI_MAP[event_toggle]) else False)
if _is_stop_event_served(event_datetime, event_status, mdi):
if previous_rt_stop_event_time > event_datetime:
raise InvalidArguments(
"invalid feed: stop_point's({}) time is not consistent"
.format(
get_value(get_value(arret, "emplacement"),
"code")))
previous_rt_stop_event_time = event_datetime
# Calculates effect from stop_time status list (this work is also done in kraken and has to be deleted there)
if trip_update.effect == TripEffect.MODIFIED_SERVICE.name:
trip_update.effect = get_effect_from_modification_type(
highest_st_status)
return trip_update
def _make_trip_updates(self, input_trip_update, data_time):
"""
If trip_update.stop_time_updates is not a strict ending subset of vj.stop_times we reject the trip update
On the other hand:
1. For the stop point present in trip_update.stop_time_updates we create a trip_update merging informations
with that of navitia stop
2. For the first stop point absent in trip_update.stop_time_updates we create a stop_time_update
with no delay for that stop
"""
vjs = self._get_navitia_vjs(input_trip_update.trip,
data_time=data_time)
trip_updates = []
for vj in vjs:
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
is_tu_valid = True
vj_stop_order = len(vj.navitia_vj.get('stop_times', [])) - 1
for vj_stop, tu_stop in itertools.izip_longest(
reversed(vj.navitia_vj.get('stop_times', [])),
reversed(input_trip_update.stop_time_update)):
if vj_stop is None:
is_tu_valid = False
break
vj_stop_point = vj_stop.get('stop_point')
if vj_stop_point is None:
is_tu_valid = False
break
if tu_stop is not None:
if self._get_stop_code(vj_stop_point) != tu_stop.stop_id:
is_tu_valid = False
break
tu_stop.stop_sequence = vj_stop_order
st_update = self._make_stoptime_update(
tu_stop, vj_stop_point)
if st_update is not None:
trip_update.stop_time_updates.append(st_update)
else:
#Initialize stops absent in trip_updates but present in vj
st_update = self._init_stop_update(vj_stop_point,
vj_stop_order)
if st_update is not None:
trip_update.stop_time_updates.append(st_update)
vj_stop_order -= 1
if is_tu_valid:
#Since vj.stop_times are managed in reversed order, we re sort stop_time_updates by order.
trip_update.stop_time_updates.sort(
cmp=lambda x, y: cmp(x.order, y.order))
trip_updates.append(trip_update)
else:
self.log.error(
'stop_time_update do not match with stops in navitia for trip : {} timestamp: {}'
.format(input_trip_update.trip.trip_id,
calendar.timegm(data_time.utctimetuple())))
record_internal_failure(
'stop_time_update do not match with stops in navitia',
contributor=self.contributor)
del trip_update.stop_time_updates[:]
return trip_updates
def _make_trip_update(self, vj, json_train):
"""
create the new TripUpdate object
Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md
"""
logger = logging.getLogger(__name__)
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
trip_message_id = get_value(json_train, 'idMotifInterneReference', nullable=True)
if trip_message_id:
trip_update.message = self.message_handler.get_message(index=trip_message_id)
trip_status = get_value(json_train, 'statutOperationnel')
if trip_status == 'SUPPRIMEE':
# the whole trip is deleted
trip_update.status = 'delete'
trip_update.stop_time_updates = []
return trip_update
elif trip_status == 'AJOUTEE':
# the trip is created from scratch
# not handled yet
self._record_and_log(logger, 'nouvelleVersion/statutOperationnel == "AJOUTEE" is not handled (yet)')
return trip_update
# all other status is considered an 'update' of the trip
trip_update.status = 'update'
pdps = _retrieve_interesting_pdp(get_value(json_train, 'listePointDeParcours'))
# manage realtime information stop_time by stop_time
for pdp in pdps:
# retrieve navitia's stop_time information corresponding to the current COTS pdp
nav_st, log_dict = self._get_navitia_stop_time(pdp, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict['log'], contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology', extra=log_dict)
if nav_st is None:
continue
nav_stop = nav_st.get('stop_point', {})
st_update = model.StopTimeUpdate(nav_stop)
trip_update.stop_time_updates.append(st_update)
# using the message from departure-time in priority, if absent fallback on arrival-time's message
st_message_id = get_value(pdp, 'idMotifInterneDepartReference', nullable=True)
if not st_message_id:
st_message_id = get_value(pdp, 'idMotifInterneArriveeReference', nullable=True)
if st_message_id:
st_update.message = self.message_handler.get_message(index=st_message_id)
_status_map = {'Arrivee': 'arrival_status', 'Depart': 'departure_status'}
_delay_map = {'Arrivee': 'arrival_delay', 'Depart': 'departure_delay'}
# compute realtime information and fill st_update for arrival and departure
for arrival_departure_toggle in ['Arrivee', 'Depart']:
cots_traveler_time = get_value(pdp,
'horaireVoyageur{}'.format(arrival_departure_toggle),
nullable=True)
if cots_traveler_time is None:
continue
cots_stop_time_status = get_value(cots_traveler_time,
'statutCirculationOPE',
nullable=True)
if cots_stop_time_status is None:
# if no cots_stop_time_status, it is considered an 'update' of the stop_time
# (can be a delay, back to normal, normal, ...)
cots_ref_planned = get_value(pdp,
'sourceHoraireProjete{}Reference'.format(
arrival_departure_toggle),
nullable=True)
cots_planned_stop_times = get_value(pdp,
'listeHoraireProjete{}'.format(arrival_departure_toggle),
nullable=True)
cots_planned_stop_time = _retrieve_projected_time(cots_ref_planned, cots_planned_stop_times)
if cots_planned_stop_time is None:
continue
cots_delay = get_value(cots_planned_stop_time, 'pronosticIV', nullable=True)
if cots_delay is None:
continue
setattr(st_update, _status_map[arrival_departure_toggle], 'update')
setattr(st_update, _delay_map[arrival_departure_toggle], as_duration(cots_delay))
elif cots_stop_time_status == 'SUPPRESSION':
# partial delete
setattr(st_update, _status_map[arrival_departure_toggle], 'delete')
elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT':
# stop_time is replaced by another one
self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == '
'"{}" is not handled completely (yet), only removal'
.format(cots_stop_time_status))
setattr(st_update, _status_map[arrival_departure_toggle], 'delete')
elif cots_stop_time_status == 'CREATION':
# new stop_time added
self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == '
'"{}" is not handled (yet)'.format(cots_stop_time_status))
elif cots_stop_time_status == 'DETOURNEMENT':
# new stop_time added also?
self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == '
'"{}" is not handled (yet)'.format(cots_stop_time_status))
else:
raise InvalidArguments('invalid value {} for field horaireVoyageur{}/statutCirculationOPE'.
format(cots_stop_time_status, arrival_departure_toggle))
return trip_update
def setup_database():
"""
we create two realtime_updates with the same vj but for different date
and return a vj for navitia
"""
with app.app_context():
vj1 = model.VehicleJourney(
{
'trip': {
'id': 'vj:1'
},
'stop_times': [{
'utc_arrival_time': time(9, 0),
'stop_point': {
'stop_area': {
'timezone': 'Europe/Paris'
}
}
}]
}, utc.localize(datetime(2015, 11, 4, 8, 0, 0)),
utc.localize(datetime(2015, 11, 4, 10, 0, 0)))
vj2 = model.VehicleJourney(
{
'trip': {
'id': 'vj:2'
},
'stop_times': [{
'utc_arrival_time': time(9, 0),
'stop_point': {
'stop_area': {
'timezone': 'Europe/Paris'
}
}
}]
}, utc.localize(datetime(2015, 11, 4, 8, 0, 0)),
utc.localize(datetime(2015, 11, 4, 10, 0, 0)))
vj3 = model.VehicleJourney(
{
'trip': {
'id': 'vj:3'
},
'stop_times': [{
'utc_arrival_time': time(9, 0),
'stop_point': {
'stop_area': {
'timezone': 'Europe/Paris'
}
}
}]
}, utc.localize(datetime(2015, 11, 4, 8, 0, 0)),
utc.localize(datetime(2015, 11, 4, 10, 0, 0)))
tu1 = model.TripUpdate(vj1, contributor='realtime.cots')
tu2 = model.TripUpdate(vj2, contributor='realtime.cots')
tu3 = model.TripUpdate(vj3, contributor='realtime.sherbrooke')
rtu1 = model.RealTimeUpdate(None, 'cots', 'realtime.cots')
rtu1.created_at = datetime(2015, 11, 4, 6, 32)
rtu1.trip_updates.append(tu1)
model.db.session.add(rtu1)
rtu2 = model.RealTimeUpdate(None, 'cots', contributor='realtime.cots')
rtu2.created_at = datetime(2015, 11, 4, 7, 32)
rtu2.trip_updates.append(tu2)
model.db.session.add(rtu2)
rtu3 = model.RealTimeUpdate(None,
'gtfs-rt',
contributor='realtime.sherbrooke')
rtu3.created_at = datetime(2015, 11, 4, 7, 42)
rtu3.trip_updates.append(tu3)
model.db.session.add(rtu3)
rtu4 = model.RealTimeUpdate(
None,
connector='gtfs-rt',
contributor='realtime.sherbrooke',
status='KO',
error='No new information destinated to navitia for this gtfs-rt')
rtu4.created_at = datetime(2015, 11, 4, 7, 52)
model.db.session.add(rtu4)
model.db.session.commit()
def _make_trip_update(self,
json_train,
vj,
action_on_trip=ActionOnTrip.NOT_ADDED.name):
"""
create the new TripUpdate object
Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md
"""
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
trip_message_id = get_value(json_train,
'idMotifInterneReference',
nullable=True)
if trip_message_id:
trip_update.message = self.message_handler.get_message(
index=trip_message_id)
cots_company_id = get_value(json_train,
'codeCompagnieTransporteur',
nullable=True) or DEFAULT_COMPANY_ID
trip_update.company_id = self._get_navitia_company(cots_company_id)
trip_status = get_value(json_train, 'statutOperationnel')
if trip_status == TripStatus.SUPPRIMEE.name:
# the whole trip is deleted
trip_update.status = ModificationType.delete.name
trip_update.stop_time_updates = []
trip_update.effect = TripEffect.NO_SERVICE.name
return trip_update
elif action_on_trip != ActionOnTrip.NOT_ADDED.name:
# the trip is created from scratch
trip_update.effect = TripEffect.ADDITIONAL_SERVICE.name
trip_update.status = ModificationType.add.name
cots_physical_mode = get_value(json_train,
'indicateurFer',
nullable=True)
trip_update.physical_mode_id = self._get_navitia_physical_mode(
cots_physical_mode)
trip_update.headsign = get_value(json_train,
'numeroCourse',
nullable=True)
# all other status is considered an 'update' of the trip_update and effect is calculated
# from stop_time status list. This part is also done in kraken and is to be deleted later
# Ordered stop_time status= 'nochange', 'add', 'delete', 'update'
# 'nochange' or 'update' -> SIGNIFICANT_DELAYS, add -> MODIFIED_SERVICE, delete = DETOUR
else:
trip_update.status = ModificationType.update.name
trip_update.effect = TripEffect.MODIFIED_SERVICE.name
# Initialize stop_time status to nochange
highest_st_status = ModificationType.none.name
pdps = _retrieve_interesting_pdp(
get_value(json_train, 'listePointDeParcours'))
# this variable is used to memoize the last stop_time's departure in order to check the stop_time consistency
# ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure
last_stop_time_depart = None
# manage realtime information stop_time by stop_time
for pdp in pdps:
# retrieve navitia's stop_point corresponding to the current COTS pdp
nav_stop, log_dict = self._get_navitia_stop_point(
pdp, vj.navitia_vj)
projected_stop_time = {
'Arrivee': None,
'Depart': None
} # used to check consistency
if log_dict:
record_internal_failure(log_dict['log'],
contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology', extra=log_dict)
if nav_stop is None:
continue
st_update = model.StopTimeUpdate(nav_stop)
trip_update.stop_time_updates.append(st_update)
# using the message from departure-time in priority, if absent fallback on arrival-time's message
st_message_id = get_value(pdp,
'idMotifInterneDepartReference',
nullable=True)
if not st_message_id:
st_message_id = get_value(pdp,
'idMotifInterneArriveeReference',
nullable=True)
if st_message_id:
st_update.message = self.message_handler.get_message(
index=st_message_id)
_status_map = {
'Arrivee': 'arrival_status',
'Depart': 'departure_status'
}
_delay_map = {
'Arrivee': 'arrival_delay',
'Depart': 'departure_delay'
}
_stop_event_datetime_map = {
'Arrivee': 'arrival',
'Depart': 'departure'
}
# compute realtime information and fill st_update for arrival and departure
for arrival_departure_toggle in ['Arrivee', 'Depart']:
cots_traveler_time = get_value(
pdp,
'horaireVoyageur{}'.format(arrival_departure_toggle),
nullable=True)
if cots_traveler_time is None:
continue
cots_stop_time_status = get_value(cots_traveler_time,
'statutCirculationOPE',
nullable=True)
if cots_stop_time_status is None:
# if no cots_stop_time_status, it is considered an 'update' of the stop_time
# (can be a delay, back to normal, normal, ...)
cots_base_datetime = _retrieve_stop_event_datetime(
cots_traveler_time)
if cots_base_datetime:
projected_stop_time[
arrival_departure_toggle] = cots_base_datetime
cots_delay = _retrieve_stop_event_delay(
pdp, arrival_departure_toggle)
if cots_delay is not None:
# It's an update only if there is delay
projected_stop_time[
arrival_departure_toggle] += cots_delay
setattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.update.name)
setattr(st_update,
_delay_map[arrival_departure_toggle],
cots_delay)
# otherwise nothing to do (status none, delay none, time none)
elif cots_stop_time_status == 'SUPPRESSION':
# partial delete
setattr(st_update, _status_map[arrival_departure_toggle],
ModificationType.delete.name)
elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT':
# stop_time is replaced by another one
setattr(st_update, _status_map[arrival_departure_toggle],
ModificationType.deleted_for_detour.name)
elif cots_stop_time_status in ['CREATION', 'DETOURNEMENT']:
# new stop_time added
cots_base_datetime = _retrieve_stop_event_datetime(
cots_traveler_time)
if cots_base_datetime:
projected_stop_time[
arrival_departure_toggle] = cots_base_datetime
cots_delay = _retrieve_stop_event_delay(
pdp, arrival_departure_toggle)
if cots_delay is not None:
projected_stop_time[
arrival_departure_toggle] += cots_delay
setattr(st_update,
_stop_event_datetime_map[arrival_departure_toggle],
projected_stop_time[arrival_departure_toggle])
# delay already added to stop_event datetime
setattr(st_update, _delay_map[arrival_departure_toggle],
None)
if cots_stop_time_status == 'CREATION':
# pure add
setattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.add.name)
elif cots_stop_time_status == 'DETOURNEMENT':
# add to replace another stop_time
setattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.added_for_detour.name)
else:
raise InvalidArguments(
'invalid value {} for field horaireVoyageur{}/statutCirculationOPE'
.format(cots_stop_time_status,
arrival_departure_toggle))
arr_dep_status = getattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.none.name)
highest_st_status = get_higher_status(highest_st_status,
arr_dep_status)
self._check_stop_time_consistency(
last_stop_time_depart,
projected_stop_time,
pdp_code='-'.join(pdp[key] for key in ['cr', 'ci', 'ch']))
last_stop_time_depart = projected_stop_time['Depart']
# Calculates effect from stop_time status list(this work is also done in kraken and has to be deleted)
if trip_update.effect == TripEffect.MODIFIED_SERVICE.name:
trip_update.effect = get_effect_by_stop_time_status(
highest_st_status)
return trip_update
def setup_database():
"""
we create two realtime_updates with the same vj but for different date
and return a vj for navitia
"""
with app.app_context():
vj1 = model.VehicleJourney(
{
"trip": {
"id": "vj:1"
},
"stop_times": [{
"utc_arrival_time": time(9, 0),
"stop_point": {
"stop_area": {
"timezone": "Europe/Paris"
}
}
}],
},
datetime(2015, 11, 4, 8, 0, 0),
datetime(2015, 11, 4, 10, 0, 0),
)
vj2 = model.VehicleJourney(
{
"trip": {
"id": "vj:2"
},
"stop_times": [{
"utc_arrival_time": time(9, 0),
"stop_point": {
"stop_area": {
"timezone": "Europe/Paris"
}
}
}],
},
datetime(2015, 11, 4, 8, 0, 0),
datetime(2015, 11, 4, 10, 0, 0),
)
vj3 = model.VehicleJourney(
{
"trip": {
"id": "vj:3"
},
"stop_times": [{
"utc_arrival_time": time(9, 0),
"stop_point": {
"stop_area": {
"timezone": "Europe/Paris"
}
}
}],
},
datetime(2015, 11, 4, 8, 0, 0),
datetime(2015, 11, 4, 10, 0, 0),
)
tu1 = model.TripUpdate(vj1, contributor_id=COTS_CONTRIBUTOR_ID)
tu2 = model.TripUpdate(vj2, contributor_id=COTS_CONTRIBUTOR_ID)
tu3 = model.TripUpdate(vj3, contributor_id=GTFS_CONTRIBUTOR_ID)
rtu1 = make_rt_update(None, COTS_CONTRIBUTOR_ID, status="OK")
rtu1.created_at = datetime(2015, 11, 4, 6, 32)
rtu1.updated_at = datetime(2015, 11, 4, 6,
32) # mock creation, no update done
rtu1.trip_updates.append(tu1)
model.db.session.add(rtu1)
rtu2 = make_rt_update(None,
contributor_id=COTS_CONTRIBUTOR_ID,
status="OK")
rtu2.created_at = datetime(2015, 11, 4, 7, 32)
rtu2.updated_at = datetime(2015, 11, 4, 7, 32)
rtu2.trip_updates.append(tu2)
model.db.session.add(rtu2)
rtu3 = make_rt_update(None,
contributor_id=GTFS_CONTRIBUTOR_ID,
status="OK")
rtu3.created_at = datetime(2015, 11, 4, 7, 42)
rtu3.updated_at = datetime(2015, 11, 4, 7, 42)
rtu3.trip_updates.append(tu3)
model.db.session.add(rtu3)
rtu4 = save_rt_data_with_error(
None,
contributor_id=GTFS_CONTRIBUTOR_ID,
error="No new information destined for navitia on rt.vroumvroum",
is_reprocess_same_data_allowed=False,
)
rtu4.created_at = datetime(2015, 11, 4, 7, 52)
rtu4.updated_at = datetime(2015, 11, 4, 7, 52)
model.db.session.add(rtu4)
rtu5 = make_rt_update(None,
contributor_id=GTFS_CONTRIBUTOR_DB_ID,
status="OK")
rtu5.created_at = datetime(2015, 11, 4, 8, 2)
rtu5.updated_at = datetime(2015, 11, 4, 8, 2)
model.db.session.add(rtu5)
rtu6 = make_rt_update(None,
contributor_id=COTS_CONTRIBUTOR_DB_ID,
status="OK")
rtu6.created_at = datetime(2015, 11, 4, 8, 12)
rtu6.updated_at = datetime(2015, 11, 4, 8, 12)
model.db.session.add(rtu6)
rtu_piv = make_rt_update(None,
contributor_id=PIV_CONTRIBUTOR_ID,
status="OK")
rtu_piv.created_at = datetime(2015, 11, 4, 8, 17)
rtu_piv.updated_at = datetime(2015, 11, 4, 8, 17)
model.db.session.add(rtu_piv)
model.db.session.commit()
def _build_trip_update(self, feed):
feed_vj = self._get_vj(feed)
if feed_vj is None:
self.log.info(
"Missing object in SIRI-ET-XML-TN feed to evaluate the real-time information; the feed should contain 'EstimatedVehicleJourney' XML object; no 'TripUpdate' will be created"
)
return None
vj_name = self._get_vj_name(feed_vj)
line_ref = self._get_line_ref(feed_vj)
origin_dt = self._get_origin_dt(feed_vj)
since_dt = origin_dt - timedelta(minutes=60)
until_dt = origin_dt + timedelta(minutes=60)
if vj_name is None or line_ref is None or origin_dt is None:
self.log.info(
"Missing information to identify the impacted vehicle journey in SIRI-ET-XML-TN; the feed should contain: 'VehicleJourneyName', 'LineRef' and 'OriginAimedDepartureTime'; no 'TripUpdate' will be created"
)
return None
vj = self._get_navitia_vj(vj_name, line_ref, since_dt, until_dt)
if vj is None:
self.log.info(
"Failed to find the vehicle journey in navitia from 'VehicleJourneyName={}', 'LineRef={}' and 'OriginAimedDepartureTime={}'; no 'TripUpdate' will be created"
.format(vj_name, line_ref, origin_dt))
return None
trip_update = model.TripUpdate(vj=vj,
contributor_id=self.contributor.id)
cancellation = self._get_cancellation(feed_vj)
extra_journey = self._get_extra_journey(feed_vj)
# Trip cancelled, no more processing is needed
if cancellation:
trip_update.effect = TripEffect.NO_SERVICE.name
trip_update.status = ModificationType.delete.name
return trip_update
# If neither Cancellation nor ExtraJourney is true,
# then we only handle delayed stop times
if not cancellation and not extra_journey:
trip_update.effect = TripEffect.SIGNIFICANT_DELAYS.name
trip_update.status = ModificationType.update.name
st_idx = 0
for feed_st in self._iter_stop_times(feed_vj):
st_stop_point_ref = self._get_st_stop_point_ref(feed_st)
st_departure_dt = self._get_st_departure_dt(feed_st)
st_arrival_dt = self._get_st_arrival_dt(feed_st)
st_cancellation = self._get_st_cancellation(feed_st)
st_extra_call = self._get_st_extra_call(feed_st)
st_platform_traversal = self._get_st_platform_traversal(
feed_st)
if st_platform_traversal:
# The stop time is ignored
continue
if st_cancellation or st_extra_call:
# We do not process (yet!) stop_time deletion or addition
self.log.info(
"Deletion or addition of stops are not yet supported; no 'TripUpdate' will be created"
)
return None
st = vj.navitia_vj.get("stop_times")[st_idx]
sp = st.get("stop_point")
base_sp = sp.get("id")
if st_stop_point_ref != base_sp:
# Stop points from base schedule and feed don't match
self.log.info(
"Stop from base schedule '{}' don't match the one '{}' in SIRI-ET-XML-TN feed; no 'TripUpdate' will be created"
.format(st_stop_point_ref, base_sp))
return None
stu = model.StopTimeUpdate(sp)
departure_delay = st_departure_dt - datetime.combine(
origin_dt.date(), st.get("utc_departure_time"))
stu.update_departure(st_departure_dt, departure_delay,
ModificationType.update.name)
arrival_delay = st_arrival_dt - datetime.combine(
origin_dt.date(), st.get("utc_arrival_time"))
stu.update_arrival(st_arrival_dt, arrival_delay,
ModificationType.update.name)
trip_update.stop_time_updates.append(stu)
st_idx += 1
return trip_update