def _build_one_stop_time_update(self, gtfsrt_stop, nav_vj, previous_rt_stop_event_time):
"""
Build a stop time update from given GTFSRT STU, navitia VJ, and
check consistency versus previous stop-event datetime (should be ascending)
:param gtfsrt_stop: GTFS StopTimeUpdate to convert
:param nav_vj: Navitia VJ to be used to retrieve information on stop
:param previous_rt_stop_event_time: Previous stop-event datetime
:return: a couple (converted STU, last_stop_event_datetime) if conversion was a success and consistent
"""
# retrieve navitia's stop_point corresponding to the current gtfsrt stop
nav_stop, log_dict = self._get_navitia_stop_point(gtfsrt_stop, nav_vj)
if log_dict:
record_internal_failure(log_dict["log"], contributor=self.contributor.id)
log_dict.update({"contributor": self.contributor.id})
self.log.info("metrology", extra=log_dict)
if nav_stop is None:
# TODO: log and count in NR
return None, None # simply skip stop_times at unknown stop points
st_update = model.StopTimeUpdate(nav_stop)
for event_toggle in ["arrival", "departure"]:
previous_rt_stop_event_time = _fill_and_check_one_stop_event(
st_update=st_update,
event_toggle=event_toggle,
gtfsrt_stop=gtfsrt_stop,
previous_rt_stop_event_time=previous_rt_stop_event_time,
)
return st_update, previous_rt_stop_event_time
def _get_navitia_vj(self, piv_key, train_date, ads, is_trip_addition):
self.log.debug("searching for vj {} in navitia".format(piv_key))
# large filter on date mostly to ensure only base-schedule VJ are requested
filter = 'vehicle_journey.has_code("rt_piv", "{}")'.format(piv_key)
train_datetime = datetime.datetime.combine(train_date,
datetime.time(0, 0, 0))
since_dt = train_datetime - datetime.timedelta(days=1)
until_dt = train_datetime + datetime.timedelta(days=2)
navitia_vjs = self._request_navitia_vehicle_journeys(filter,
since_dt,
until_dt,
depth=2,
show_codes=True)
if not navitia_vjs:
# Last PIV information is always right, so if the VJ doesn't exist, it's an ADD (no matter feed content)
navitia_vjs = [_make_navitia_empty_vj(piv_key)]
vj = None
if len(navitia_vjs) != 1:
self.log.info(
"Can not match a unique train for key {}".format(piv_key))
record_internal_failure("no unique train",
contributor=self.contributor.id)
else:
navitia_vj = navitia_vjs[0]
try:
base_vs_rt_error_margin = datetime.timedelta(hours=1)
vj_base_start = _get_first_stop_base_datetime(
ads, "depart", skip_fully_added_stops=not is_trip_addition)
# not even a single stop is a base-schedule stop
if not vj_base_start:
raise InvalidArguments(
"Whole trip is specified as pre-existing, but no stop is specified as pre-existing"
)
vj = model.VehicleJourney(
navitia_vj,
vj_base_start - base_vs_rt_error_margin,
vj_base_start + base_vs_rt_error_margin,
vj_start_dt=vj_base_start,
)
except InvalidArguments as i:
raise i
except Exception as e:
self.log.exception(
"Error while creating kirin VJ of {}: {}".format(
navitia_vjs[0].get("id"), e))
record_internal_failure("Error while creating kirin VJ",
contributor=self.contributor.id)
if not vj:
raise ObjectNotFound("no train found for key {}".format(piv_key))
return vj
def _get_navitia_vjs(self, headsign_str, since_dt, until_dt):
"""
Search for navitia's vehicle journeys with given headsigns, in the period provided
"""
log = logging.getLogger(__name__)
vjs = {}
# to get the date of the vj we use the start/end of the vj + some tolerance
# since the SNCF data and navitia data might not be synchronized
extended_since_dt = since_dt - timedelta(hours=1)
extended_until_dt = until_dt + timedelta(hours=1)
# using a set to deduplicate
# one headsign_str (ex: "96320/1") can lead to multiple headsigns (ex: ["96320", "96321"])
# but most of the time (if not always) they refer to the same VJ
# (the VJ switches headsign along the way).
# So we do one VJ search for each headsign to ensure we get it, then deduplicate VJs
for train_number in headsigns(headsign_str):
log.debug('searching for vj {} on {} in navitia'.format(
train_number, since_dt))
navitia_vjs = self.navitia.vehicle_journeys(
q={
'headsign': train_number,
'since': to_navitia_str(extended_since_dt),
'until': to_navitia_str(extended_until_dt),
'depth':
'2', # we need this depth to get the stoptime's stop_area
'show_codes':
'true' # we need the stop_points CRCICH codes
})
if not navitia_vjs:
logging.getLogger(__name__).info(
'impossible to find train {t} on [{s}, {u}['.format(
t=train_number,
s=extended_since_dt,
u=extended_until_dt))
record_internal_failure('missing train',
contributor=self.contributor)
for nav_vj in navitia_vjs:
vj = model.VehicleJourney(nav_vj, since_dt.date())
vjs[nav_vj['id']] = vj
if not vjs:
raise ObjectNotFound(
'no train found for headsign(s) {}'.format(headsign_str))
return vjs.values()
def _get_vjs(self, xml_train):
log = logging.getLogger(__name__)
train_numbers = headsigns(get_value(xml_train, 'NumeroTrain'))
# to get the date of the vj we use the start/end of the vj + some tolerance
# since the ire data and navitia data might not be synchronized
vj_start = as_date(
get_value(xml_train, 'OrigineTheoriqueTrain/DateHeureDepart'))
since = vj_start - timedelta(hours=1)
vj_end = as_date(
get_value(xml_train, 'TerminusTheoriqueTrain/DateHeureTerminus'))
until = vj_end + timedelta(hours=1)
vjs = {}
for train_number in train_numbers:
log.debug('searching for vj {} on {} in navitia'.format(
train_number, vj_start))
navitia_vjs = self.navitia.vehicle_journeys(
q={
'headsign': train_number,
'since': to_str(since),
'until': to_str(until),
'depth':
'2', # we need this depth to get the stoptime's stop_area
'show_codes':
'true' # we need the stop_points CRCICH codes
})
if not navitia_vjs:
logging.getLogger(__name__).info(
'impossible to find train {t} on [{s}, {u}['.format(
t=train_number, s=since, u=until))
record_internal_failure('missing train',
contributor=self.contributor)
for nav_vj in navitia_vjs:
vj = model.VehicleJourney(nav_vj, vj_start.date())
vjs[nav_vj['id']] = vj
if not vjs:
raise ObjectNotFound(
'no train found for headsigns {}'.format(train_numbers))
return vjs.values()
def _get_navitia_vj(self, trip_id, since_dt, until_dt):
filter = "vehicle_journey.has_code({}, {})".format(self.stop_code_key, trip_id)
navitia_vjs = self._request_navitia_vehicle_journeys(filter, since_dt, until_dt, depth=2)
if not navitia_vjs:
self.log.info("impossible to find vj {t} on [{s}, {u}]".format(t=trip_id, s=since_dt, u=until_dt))
record_internal_failure("missing vj", contributor=self.contributor.id)
return None
if len(navitia_vjs) > 1:
vj_ids = [vj.get("id") for vj in navitia_vjs]
self.log.info(
"too many vjs found for {t} on [{s}, {u}]: {ids}".format(
t=trip_id, s=since_dt, u=until_dt, ids=vj_ids
)
)
record_internal_failure("duplicate vjs", contributor=self.contributor.id)
return None
nav_vj = navitia_vjs[0]
vj = None
try:
vj = model.VehicleJourney(nav_vj, since_dt, until_dt)
except Exception as e:
self.log.exception("Error while creating kirin VJ of {}: {}".format(nav_vj.get("id"), e))
record_internal_failure("Error while creating kirin VJ", contributor=self.contributor.id)
return vj
def _make_db_vj(self, vj_source_code, since_dt, until_dt):
"""
Search for navitia's vehicle journeys with given code, in the period provided
:param vj_source_code: the code to search for
:param since_dt: naive UTC datetime that starts the search period.
Typically the supposed datetime of first base-schedule stop_time.
:param until_dt: naive UTC datetime that ends the search period.
Typically the supposed datetime of last base-schedule stop_time.
"""
if since_dt.tzinfo is not None or until_dt.tzinfo is not None:
raise InternalException(
"Invalid datetime provided: must be naive (and UTC)")
navitia_vjs = self.navitia.vehicle_journeys(
q={
"filter":
"vehicle_journey.has_code({}, {})".format(
self.stop_code_key, vj_source_code),
"since":
to_navitia_utc_str(since_dt),
"until":
to_navitia_utc_str(until_dt),
"depth":
"2", # we need this depth to get the stoptime's stop_area
})
if not navitia_vjs:
self.log.info("impossible to find vj {t} on [{s}, {u}]".format(
t=vj_source_code, s=since_dt, u=until_dt))
record_internal_failure("missing vj",
contributor=self.contributor.id)
return []
if len(navitia_vjs) > 1:
vj_ids = [vj.get("id") for vj in navitia_vjs]
self.log.info(
"too many vjs found for {t} on [{s}, {u}]: {ids}".format(
t=vj_source_code, s=since_dt, u=until_dt, ids=vj_ids))
record_internal_failure("duplicate vjs",
contributor=self.contributor.id)
return []
nav_vj = navitia_vjs[0]
try:
vj = model.VehicleJourney(nav_vj, since_dt, until_dt)
return [vj]
except Exception as e:
self.log.exception(
"Error while creating kirin VJ of {}: {}".format(
nav_vj.get("id"), e))
record_internal_failure("Error while creating kirin VJ",
contributor=self.contributor.id)
return []
def _make_db_vj(self, vj_source_code, since_dt, until_dt):
"""
Search for navitia's vehicle journeys with given code, in the period provided
:param vj_source_code: the code to search for
:param since_dt: naive UTC datetime that starts the search period.
Typically the supposed datetime of first base-schedule stop_time.
:param until_dt: naive UTC datetime that ends the search period.
Typically the supposed datetime of last base-schedule stop_time.
"""
filter = "vehicle_journey.has_code({}, {})".format(
self.stop_code_key, vj_source_code)
navitia_vjs = self._request_navitia_vehicle_journeys(filter,
since_dt,
until_dt,
depth=2)
if not navitia_vjs:
self.log.info("impossible to find vj {t} on [{s}, {u}]".format(
t=vj_source_code, s=since_dt, u=until_dt))
record_internal_failure("missing vj",
contributor=self.contributor.id)
return []
if len(navitia_vjs) > 1:
vj_ids = [vj.get("id") for vj in navitia_vjs]
self.log.info(
"too many vjs found for {t} on [{s}, {u}]: {ids}".format(
t=vj_source_code, s=since_dt, u=until_dt, ids=vj_ids))
record_internal_failure("duplicate vjs",
contributor=self.contributor.id)
return []
nav_vj = navitia_vjs[0]
try:
vj = model.VehicleJourney(nav_vj, since_dt, until_dt)
return [vj]
except Exception as e:
self.log.exception(
"Error while creating kirin VJ of {}: {}".format(
nav_vj.get("id"), e))
record_internal_failure("Error while creating kirin VJ",
contributor=self.contributor.id)
return []
def _make_db_vj(self, vj_source_code, utc_since_dt, utc_until_dt):
navitia_vjs = self.navitia.vehicle_journeys(
q={
'filter':
'vehicle_journey.has_code({}, {})'.format(
self.stop_code_key, vj_source_code),
'since':
to_str(utc_since_dt),
'until':
to_str(utc_until_dt),
'depth':
'2', # we need this depth to get the stoptime's stop_area
})
if not navitia_vjs:
self.log.info('impossible to find vj {t} on [{s}, {u}]'.format(
t=vj_source_code, s=utc_since_dt, u=utc_until_dt))
record_internal_failure('missing vj', contributor=self.contributor)
return []
if len(navitia_vjs) > 1:
vj_ids = [vj.get('id') for vj in navitia_vjs]
self.log.info(
'too many vjs found for {t} on [{s}, {u}]: {ids}'.format(
t=vj_source_code,
s=utc_since_dt,
u=utc_until_dt,
ids=vj_ids))
record_internal_failure('duplicate vjs',
contributor=self.contributor)
return []
nav_vj = navitia_vjs[0]
try:
vj = model.VehicleJourney(nav_vj, utc_since_dt, utc_until_dt)
return [vj]
except Exception as e:
self.log.exception(
'Error while creating kirin VJ of {}: {}'.format(
nav_vj.get('id'), e))
record_internal_failure('Error while creating kirin VJ',
contributor=self.contributor)
return []
def _make_trip_updates(self, input_trip_update, input_data_time):
"""
If trip_update.stop_time_updates is not a strict ending subset of vj.stop_times we reject the trip update
On the other hand:
1. For the stop point present in trip_update.stop_time_updates we create a trip_update merging
informations with that of navitia stop
2. For the first stop point absent in trip_update.stop_time_updates we create a stop_time_update
with no delay for that stop
"""
vjs = self._get_navitia_vjs(input_trip_update.trip,
input_data_time=input_data_time)
trip_updates = []
for vj in vjs:
trip_update = model.TripUpdate(vj=vj,
contributor_id=self.contributor.id)
highest_st_status = ModificationType.none.name
is_tu_valid = True
vj_stop_order = len(vj.navitia_vj.get("stop_times", [])) - 1
for vj_stop, tu_stop in itertools.zip_longest(
reversed(vj.navitia_vj.get("stop_times", [])),
reversed(input_trip_update.stop_time_update)):
if vj_stop is None:
is_tu_valid = False
break
vj_stop_point = vj_stop.get("stop_point")
if vj_stop_point is None:
is_tu_valid = False
break
if tu_stop is not None:
if self._get_stop_code(vj_stop_point) != tu_stop.stop_id:
is_tu_valid = False
break
tu_stop.stop_sequence = vj_stop_order
st_update = _make_stoptime_update(tu_stop, vj_stop_point)
if st_update is not None:
trip_update.stop_time_updates.append(st_update)
else:
# Initialize stops absent in trip_updates but present in vj
st_update = _init_stop_update(vj_stop_point, vj_stop_order)
if st_update is not None:
trip_update.stop_time_updates.append(st_update)
for status in [
st_update.departure_status, st_update.arrival_status
]:
highest_st_status = get_higher_status(
highest_st_status, status)
vj_stop_order -= 1
if is_tu_valid:
# Since vj.stop_times are managed in reversed order, we re sort stop_time_updates by order.
trip_update.stop_time_updates.sort(key=lambda x: x.order)
trip_update.effect = get_effect_from_modification_type(
highest_st_status)
trip_updates.append(trip_update)
else:
self.log.warning(
"stop_time_update do not match with stops in navitia for trip : {} timestamp: {}"
.format(input_trip_update.trip.trip_id,
calendar.timegm(input_data_time.utctimetuple())))
record_internal_failure(
"stop_time_update do not match with stops in navitia",
contributor=self.contributor.id)
del trip_update.stop_time_updates[:]
return trip_updates
def _make_trip_update(self, json_train, vj):
trip_update = model.TripUpdate(vj=vj,
contributor_id=self.contributor.id)
trip_update.headsign = json_train.get("numero")
company_id = json_train.get("operateur").get("codeOperateur")
trip_update.company_id = self._get_navitia_company_id(company_id)
physical_mode = json_train.get("modeTransport").get("typeMode")
trip_update.physical_mode_id = self._get_navitia_physical_mode_id(
physical_mode)
trip_status_type = trip_piv_status_to_effect.get(
json_train.get("evenement").get("type"), TripEffect.UNDEFINED)
trip_update.message = json_train.get("evenement").get("texte")
trip_update.effect = trip_status_type.name
if trip_status_type == TripEffect.NO_SERVICE:
# the whole trip is deleted
trip_update.status = ModificationType.delete.name
trip_update.stop_time_updates = []
return trip_update
elif trip_status_type == TripEffect.ADDITIONAL_SERVICE:
trip_update.status = ModificationType.add.name
else:
trip_update.status = ModificationType.update.name
highest_st_status = ModificationType.none.name
ads = get_value(get_value(json_train, "listeArretsDesserte"), "arret")
# this variable is used to store the last stop_time's departure in order to check the stop_time consistency
# ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure
previous_rt_stop_event_time = datetime.datetime.utcfromtimestamp(0)
for arret in ads:
# retrieve navitia's stop_point corresponding to the current PIV ad
nav_stop, log_dict = self._get_navitia_stop_point(
arret, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict["log"],
contributor=self.contributor.id)
log_dict.update({"contributor": self.contributor.id})
logging.getLogger(__name__).info("metrology", extra=log_dict)
if nav_stop is None:
continue # simply skip stop_times at unknown stop areas
st_update = model.StopTimeUpdate(nav_stop)
trip_update.stop_time_updates.append(st_update)
st_update.message = _get_message(arret)
for event_toggle in ["arrivee", "depart"]:
event = get_value(arret, event_toggle, nullable=True)
if event is None:
continue
piv_disruption = event.get("evenement", {})
piv_event_status = event.get("statutModification",
piv_disruption.get("type", None))
if not piv_event_status or piv_event_status in MANAGED_STOP_EVENTS:
piv_event_datetime = get_value(event,
"dateHeureReelle",
nullable=True)
event_datetime = str_to_utc_naive_dt(
piv_event_datetime) if piv_event_datetime else None
if event_datetime:
setattr(st_update,
STOP_EVENT_DATETIME_MAP[event_toggle],
event_datetime)
if piv_event_status in ["SUPPRESSION_PARTIELLE"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.delete.name)
elif piv_event_status in ["SUPPRESSION_DETOURNEMENT"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.deleted_for_detour.name)
elif piv_event_status in ["CREATION"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.add.name)
elif piv_event_status in ["CREATION_DETOURNEMENT"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.added_for_detour.name)
elif trip_status_type == TripEffect.ADDITIONAL_SERVICE:
# In this case we want to create schedules taking into account the delay
# without adding the delay a second time
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.add.name)
elif piv_event_status in [
"RETARD_OBSERVE", "RETARD_PROJETE"
]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.update.name)
if piv_disruption:
piv_event_delay = piv_disruption.get(
"retard", {}).get("duree", 0)
setattr(st_update, DELAY_MAP[event_toggle],
as_duration(piv_event_delay *
60)) # minutes
else:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.none.name)
# otherwise let those be none
else:
raise InvalidArguments(
"invalid value {s} for field {t}/statutModification or {t}/evenement/type"
.format(s=piv_event_status, t=event_toggle))
event_status = getattr(st_update, STATUS_MAP[event_toggle],
ModificationType.none.name)
highest_st_status = get_higher_status(highest_st_status,
event_status)
mdi = (arret[STOP_EVENT_MDI_MAP[event_toggle]] if arret.get(
STOP_EVENT_MDI_MAP[event_toggle]) else False)
if _is_stop_event_served(event_datetime, event_status, mdi):
if previous_rt_stop_event_time > event_datetime:
raise InvalidArguments(
"invalid feed: stop_point's({}) time is not consistent"
.format(
get_value(get_value(arret, "emplacement"),
"code")))
previous_rt_stop_event_time = event_datetime
# Calculates effect from stop_time status list (this work is also done in kraken and has to be deleted there)
if trip_update.effect == TripEffect.MODIFIED_SERVICE.name:
trip_update.effect = get_effect_from_modification_type(
highest_st_status)
return trip_update
def _get_navitia_vjs(self,
headsign_str,
since_dt,
until_dt,
action_on_trip=ActionOnTrip.NOT_ADDED.name):
"""
Search for navitia's vehicle journeys with given headsigns, in the period provided
:param headsign_str: the headsigns to search for (can be multiple expressed in one string, like "2512/3")
:param since_dt: naive UTC datetime that starts the search period.
Typically the supposed datetime of first base-schedule stop_time.
:param until_dt: naive UTC datetime that ends the search period.
Typically the supposed datetime of last base-schedule stop_time.
:param action_on_trip: action to be performed on trip. This param is used to do consistency check
"""
if (since_dt is None) or (until_dt is None):
return []
if since_dt.tzinfo is not None or until_dt.tzinfo is not None:
raise InternalException(
"Invalid datetime provided: must be naive (and UTC)")
vjs = {}
# to get the date of the vj we use the start/end of the vj + some tolerance
# since the SNCF data and navitia data might not be synchronized
extended_since_dt = since_dt - SNCF_SEARCH_MARGIN
extended_until_dt = until_dt + SNCF_SEARCH_MARGIN
# using a set to deduplicate
# one headsign_str (ex: "96320/1") can lead to multiple headsigns (ex: ["96320", "96321"])
# but most of the time (if not always) they refer to the same VJ
# (the VJ switches headsign along the way).
# So we do one VJ search for each headsign to ensure we get it, then deduplicate VJs
for train_number in headsigns(headsign_str):
self.log.debug(
"searching for vj {} during period [{} - {}] in navitia".
format(train_number, extended_since_dt, extended_until_dt))
navitia_vjs = self.navitia.vehicle_journeys(
q={
"headsign": train_number,
"since": to_navitia_utc_str(extended_since_dt),
"until": to_navitia_utc_str(extended_until_dt),
"data_freshness": "base_schedule",
"depth":
"2", # we need this depth to get the stoptime's stop_area
"show_codes":
"true", # we need the stop_points CRCICH codes
})
# Consistency check on action applied to trip
if action_on_trip == ActionOnTrip.NOT_ADDED.name:
if not navitia_vjs:
self.log.info(
"impossible to find train {t} on [{s}, {u}[".format(
t=train_number,
s=extended_since_dt,
u=extended_until_dt))
record_internal_failure("missing train",
contributor=self.contributor.id)
else:
if action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name and navitia_vjs:
raise InvalidArguments(
"Invalid action, trip {} already present in navitia".
format(train_number))
navitia_vjs = [make_navitia_empty_vj(train_number)]
for nav_vj in navitia_vjs:
try:
vj = model.VehicleJourney(nav_vj,
extended_since_dt,
extended_until_dt,
vj_start_dt=since_dt)
vjs[nav_vj["id"]] = vj
except Exception as e:
self.log.exception(
"Error while creating kirin VJ of {}: {}".format(
nav_vj.get("id"), e))
record_internal_failure("Error while creating kirin VJ",
contributor=self.contributor.id)
if not vjs:
raise ObjectNotFound(
"no train found for headsign(s) {}".format(headsign_str))
return vjs.values()
def merge_trip_updates(self, navitia_vj, db_trip_update, new_trip_update):
"""
Steps:
1. Build TripUpdate info resulting of merge
2. Adjust TripUpdate info to have it be self-consistent (call adjust_trip_update_consistency())
3. If resulting TripUpdate info are new compared to the one from previous RT info: send it
NB:
* Working with ORM objects directly: no persistence of new object before knowing it's final version and
it's not a duplicate (performance and db unicity constraints)
"""
# * Retrieve information available in the 3 StopTimeUpdate lists *
circulation_date = new_trip_update.vj.get_circulation_date()
new_stus = new_trip_update.stop_time_updates
# base-schedule STU list
nav_stus = convert_nav_stop_list_to_stu_list(navitia_vj.get("stop_times", []), circulation_date)
# last info known about STU in trip (before processing new feed):
# either from previous RT feed or base-schedule
old_stus = db_trip_update.stop_time_updates if db_trip_update else nav_stus
# * Build matches between stus in different lists *
old_to_nav = map_old_stu_to_nav_stu(old_stus=old_stus, nav_stus=nav_stus)
if old_to_nav is None:
# If nav_stus is not strictly contained into old_stus, then base-schedule was probably reloaded.
# So let's restart from scratch using base-schedule
fail_str = "Warning: previous RT result doesn't contain base-schedule"
log_str = "{f}: navitia_vj.id={id}".format(f=fail_str, id=navitia_vj.get("id"))
self.log.warning(log_str)
record_internal_failure(fail_str, contributor=self.contributor.id, reason=log_str)
old_stus = nav_stus
old_to_nav = map_old_stu_to_nav_stu(old_stus=old_stus, nav_stus=nav_stus)
new_to_old = list_match_new_stu_to_old_stu(new_stus=new_stus, old_stus=old_stus)
# * Build resulting STU list *
# keeps track of what was processed in previous STU info
old_stus_unprocessed_start = new_to_old[0][1] if new_to_old else len(old_stus)
# Old STUs before the first STU mentioned in new_stus:
# Regular behavior is to keep them as-is (considered "archived")
res_stus = old_stus[:old_stus_unprocessed_start]
# Only exception is if trip is entirely deleted: keep only stops in nav_stus
if new_trip_update.effect == TripEffect.NO_SERVICE.name:
res_stus = [
old_stus[idx] for idx in filter(lambda i: i in old_to_nav, range(0, old_stus_unprocessed_start))
]
new_stus_unprocessed_start = 0
propagated_delay = as_duration(0)
# populate with new_stus (iterating on matches and filling voids in matches-intervals)
for new_index, old_index in new_to_old:
stus_until_match, last_delay = build_stops_until_match(
match_new_index=new_index,
match_old_index=old_index,
new_stus_unprocessed_start=new_stus_unprocessed_start,
old_stus_unprocessed_start=old_stus_unprocessed_start,
new_stus=new_stus,
nav_stus=nav_stus,
old_to_nav=old_to_nav,
result_idx_offset=len(res_stus),
propagated_delay=propagated_delay,
)
res_stus.extend(stus_until_match)
propagated_delay = last_delay
# Remember progress
new_stus_unprocessed_start = new_index + 1
old_stus_unprocessed_start = old_index + 1
# Finalize populating unmatched STUs in old_stus and reversely those unmatched in new_stus
stus_until_end, _last_delay = create_intermediate_stus_before_match(
match_new_index=len(new_stus),
match_old_index=len(old_stus),
new_stus_unprocessed_start=new_stus_unprocessed_start,
old_stus_unprocessed_start=old_stus_unprocessed_start,
new_stus=new_stus,
nav_stus=nav_stus,
old_to_nav=old_to_nav,
result_idx_offset=len(res_stus),
propagated_delay=propagated_delay,
)
res_stus.extend(stus_until_end)
# adjust consistency for resulting trip_update
adjust_gtfsrt_trip_update_consistency(new_trip_update, res_stus, nav_stus)
return build_trip_update_if_modified(
db_trip_update, old_stus, new_trip_update, res_stus, self.contributor.id
)
def _make_db_vj(self, vj_source_code, since, until):
navitia_vjs = self.navitia.vehicle_journeys(
q={
'filter':
'vehicle_journey.has_code({}, {})'.format(
self.stop_code_key, vj_source_code),
'since':
to_str(since),
'until':
to_str(until),
'depth':
'2', # we need this depth to get the stoptime's stop_area
})
if not navitia_vjs:
self.log.info('impossible to find vj {t} on [{s}, {u}]'.format(
t=vj_source_code, s=since, u=until))
record_internal_failure('missing vj', contributor=self.contributor)
return []
if len(navitia_vjs) > 1:
vj_ids = [vj.get('id') for vj in navitia_vjs]
self.log.info(
'too many vjs found for {t} on [{s}, {u}]: {ids}'.format(
t=vj_source_code, s=since, u=until, ids=vj_ids))
record_internal_failure('duplicate vjs',
contributor=self.contributor)
return []
nav_vj = navitia_vjs[0]
# Now we compute the real circulate_date of VJ from since, until and vj's first stop_time
# We do this to prevent cases like pass midnight when [since, until] is too large
# we need local timezone circulate_date (and it's sometimes different from UTC date)
first_stop_time = nav_vj.get('stop_times', [{}])[0]
tzinfo = get_timezone(first_stop_time)
# 'since' and 'until' must have a timezone before being converted to local timezone
local_since = pytz.utc.localize(since).astimezone(tzinfo)
local_until = pytz.utc.localize(until).astimezone(tzinfo)
circulate_date = None
if local_since.date() == local_until.date():
circulate_date = local_since.date()
else:
arrival_time = first_stop_time['arrival_time']
# At first, we suppose that the circulate_date is local_since's date
if local_since <= tzinfo.localize(
datetime.datetime.combine(local_since.date(),
arrival_time)) <= local_until:
circulate_date = local_since.date()
elif local_since <= tzinfo.localize(
datetime.datetime.combine(local_until.date(),
arrival_time)) <= local_until:
circulate_date = local_until.date()
if circulate_date is None:
self.log.error(
'impossible to calculate the circulate date (local) of vj: {}'.
format(nav_vj.get('id')))
record_internal_failure(
'impossible to calculate the circulate date of vj',
contributor=self.contributor)
return []
try:
vj = model.VehicleJourney(nav_vj, circulate_date)
return [vj]
except Exception as e:
self.log.exception(
'Error while creating kirin VJ of {}: {}'.format(
nav_vj.get('id'), e))
record_internal_failure('Error while creating kirin VJ',
contributor=self.contributor)
return []
def _make_trip_updates(self, input_trip_update, data_time):
"""
If trip_update.stop_time_updates is not a strict ending subset of vj.stop_times we reject the trip update
On the other hand:
1. For the stop point present in trip_update.stop_time_updates we create a trip_update merging informations
with that of navitia stop
2. For the first stop point absent in trip_update.stop_time_updates we create a stop_time_update
with no delay for that stop
"""
vjs = self._get_navitia_vjs(input_trip_update.trip,
data_time=data_time)
trip_updates = []
for vj in vjs:
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
is_tu_valid = True
vj_stop_order = len(vj.navitia_vj.get('stop_times', [])) - 1
for vj_stop, tu_stop in itertools.izip_longest(
reversed(vj.navitia_vj.get('stop_times', [])),
reversed(input_trip_update.stop_time_update)):
if vj_stop is None:
is_tu_valid = False
break
vj_stop_point = vj_stop.get('stop_point')
if vj_stop_point is None:
is_tu_valid = False
break
if tu_stop is not None:
if self._get_stop_code(vj_stop_point) != tu_stop.stop_id:
is_tu_valid = False
break
tu_stop.stop_sequence = vj_stop_order
st_update = self._make_stoptime_update(
tu_stop, vj_stop_point)
if st_update is not None:
trip_update.stop_time_updates.append(st_update)
else:
#Initialize stops absent in trip_updates but present in vj
st_update = self._init_stop_update(vj_stop_point,
vj_stop_order)
if st_update is not None:
trip_update.stop_time_updates.append(st_update)
vj_stop_order -= 1
if is_tu_valid:
#Since vj.stop_times are managed in reversed order, we re sort stop_time_updates by order.
trip_update.stop_time_updates.sort(
cmp=lambda x, y: cmp(x.order, y.order))
trip_updates.append(trip_update)
else:
self.log.error(
'stop_time_update do not match with stops in navitia for trip : {} timestamp: {}'
.format(input_trip_update.trip.trip_id,
calendar.timegm(data_time.utctimetuple())))
record_internal_failure(
'stop_time_update do not match with stops in navitia',
contributor=self.contributor)
del trip_update.stop_time_updates[:]
return trip_updates
def _make_trip_update(self, vj, json_train):
"""
create the new TripUpdate object
Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md
"""
logger = logging.getLogger(__name__)
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
trip_message_id = get_value(json_train, 'idMotifInterneReference', nullable=True)
if trip_message_id:
trip_update.message = self.message_handler.get_message(index=trip_message_id)
trip_status = get_value(json_train, 'statutOperationnel')
if trip_status == 'SUPPRIMEE':
# the whole trip is deleted
trip_update.status = 'delete'
trip_update.stop_time_updates = []
return trip_update
elif trip_status == 'AJOUTEE':
# the trip is created from scratch
# not handled yet
self._record_and_log(logger, 'nouvelleVersion/statutOperationnel == "AJOUTEE" is not handled (yet)')
return trip_update
# all other status is considered an 'update' of the trip
trip_update.status = 'update'
pdps = _retrieve_interesting_pdp(get_value(json_train, 'listePointDeParcours'))
# manage realtime information stop_time by stop_time
for pdp in pdps:
# retrieve navitia's stop_time information corresponding to the current COTS pdp
nav_st, log_dict = self._get_navitia_stop_time(pdp, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict['log'], contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology', extra=log_dict)
if nav_st is None:
continue
nav_stop = nav_st.get('stop_point', {})
st_update = model.StopTimeUpdate(nav_stop)
trip_update.stop_time_updates.append(st_update)
# using the message from departure-time in priority, if absent fallback on arrival-time's message
st_message_id = get_value(pdp, 'idMotifInterneDepartReference', nullable=True)
if not st_message_id:
st_message_id = get_value(pdp, 'idMotifInterneArriveeReference', nullable=True)
if st_message_id:
st_update.message = self.message_handler.get_message(index=st_message_id)
_status_map = {'Arrivee': 'arrival_status', 'Depart': 'departure_status'}
_delay_map = {'Arrivee': 'arrival_delay', 'Depart': 'departure_delay'}
# compute realtime information and fill st_update for arrival and departure
for arrival_departure_toggle in ['Arrivee', 'Depart']:
cots_traveler_time = get_value(pdp,
'horaireVoyageur{}'.format(arrival_departure_toggle),
nullable=True)
if cots_traveler_time is None:
continue
cots_stop_time_status = get_value(cots_traveler_time,
'statutCirculationOPE',
nullable=True)
if cots_stop_time_status is None:
# if no cots_stop_time_status, it is considered an 'update' of the stop_time
# (can be a delay, back to normal, normal, ...)
cots_ref_planned = get_value(pdp,
'sourceHoraireProjete{}Reference'.format(
arrival_departure_toggle),
nullable=True)
cots_planned_stop_times = get_value(pdp,
'listeHoraireProjete{}'.format(arrival_departure_toggle),
nullable=True)
cots_planned_stop_time = _retrieve_projected_time(cots_ref_planned, cots_planned_stop_times)
if cots_planned_stop_time is None:
continue
cots_delay = get_value(cots_planned_stop_time, 'pronosticIV', nullable=True)
if cots_delay is None:
continue
setattr(st_update, _status_map[arrival_departure_toggle], 'update')
setattr(st_update, _delay_map[arrival_departure_toggle], as_duration(cots_delay))
elif cots_stop_time_status == 'SUPPRESSION':
# partial delete
setattr(st_update, _status_map[arrival_departure_toggle], 'delete')
elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT':
# stop_time is replaced by another one
self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == '
'"{}" is not handled completely (yet), only removal'
.format(cots_stop_time_status))
setattr(st_update, _status_map[arrival_departure_toggle], 'delete')
elif cots_stop_time_status == 'CREATION':
# new stop_time added
self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == '
'"{}" is not handled (yet)'.format(cots_stop_time_status))
elif cots_stop_time_status == 'DETOURNEMENT':
# new stop_time added also?
self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == '
'"{}" is not handled (yet)'.format(cots_stop_time_status))
else:
raise InvalidArguments('invalid value {} for field horaireVoyageur{}/statutCirculationOPE'.
format(cots_stop_time_status, arrival_departure_toggle))
return trip_update
def _record_and_log(self, logger, log_str):
log_dict = {'log': log_str}
record_internal_failure(log_dict['log'], contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logger.info('metrology', extra=log_dict)
def _make_trip_update(self,
json_train,
vj,
action_on_trip=ActionOnTrip.NOT_ADDED.name):
"""
create the new TripUpdate object
Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md
"""
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
trip_message_id = get_value(json_train,
'idMotifInterneReference',
nullable=True)
if trip_message_id:
trip_update.message = self.message_handler.get_message(
index=trip_message_id)
cots_company_id = get_value(json_train,
'codeCompagnieTransporteur',
nullable=True) or DEFAULT_COMPANY_ID
trip_update.company_id = self._get_navitia_company(cots_company_id)
trip_status = get_value(json_train, 'statutOperationnel')
if trip_status == TripStatus.SUPPRIMEE.name:
# the whole trip is deleted
trip_update.status = ModificationType.delete.name
trip_update.stop_time_updates = []
trip_update.effect = TripEffect.NO_SERVICE.name
return trip_update
elif action_on_trip != ActionOnTrip.NOT_ADDED.name:
# the trip is created from scratch
trip_update.effect = TripEffect.ADDITIONAL_SERVICE.name
trip_update.status = ModificationType.add.name
cots_physical_mode = get_value(json_train,
'indicateurFer',
nullable=True)
trip_update.physical_mode_id = self._get_navitia_physical_mode(
cots_physical_mode)
trip_update.headsign = get_value(json_train,
'numeroCourse',
nullable=True)
# all other status is considered an 'update' of the trip_update and effect is calculated
# from stop_time status list. This part is also done in kraken and is to be deleted later
# Ordered stop_time status= 'nochange', 'add', 'delete', 'update'
# 'nochange' or 'update' -> SIGNIFICANT_DELAYS, add -> MODIFIED_SERVICE, delete = DETOUR
else:
trip_update.status = ModificationType.update.name
trip_update.effect = TripEffect.MODIFIED_SERVICE.name
# Initialize stop_time status to nochange
highest_st_status = ModificationType.none.name
pdps = _retrieve_interesting_pdp(
get_value(json_train, 'listePointDeParcours'))
# this variable is used to memoize the last stop_time's departure in order to check the stop_time consistency
# ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure
last_stop_time_depart = None
# manage realtime information stop_time by stop_time
for pdp in pdps:
# retrieve navitia's stop_point corresponding to the current COTS pdp
nav_stop, log_dict = self._get_navitia_stop_point(
pdp, vj.navitia_vj)
projected_stop_time = {
'Arrivee': None,
'Depart': None
} # used to check consistency
if log_dict:
record_internal_failure(log_dict['log'],
contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology', extra=log_dict)
if nav_stop is None:
continue
st_update = model.StopTimeUpdate(nav_stop)
trip_update.stop_time_updates.append(st_update)
# using the message from departure-time in priority, if absent fallback on arrival-time's message
st_message_id = get_value(pdp,
'idMotifInterneDepartReference',
nullable=True)
if not st_message_id:
st_message_id = get_value(pdp,
'idMotifInterneArriveeReference',
nullable=True)
if st_message_id:
st_update.message = self.message_handler.get_message(
index=st_message_id)
_status_map = {
'Arrivee': 'arrival_status',
'Depart': 'departure_status'
}
_delay_map = {
'Arrivee': 'arrival_delay',
'Depart': 'departure_delay'
}
_stop_event_datetime_map = {
'Arrivee': 'arrival',
'Depart': 'departure'
}
# compute realtime information and fill st_update for arrival and departure
for arrival_departure_toggle in ['Arrivee', 'Depart']:
cots_traveler_time = get_value(
pdp,
'horaireVoyageur{}'.format(arrival_departure_toggle),
nullable=True)
if cots_traveler_time is None:
continue
cots_stop_time_status = get_value(cots_traveler_time,
'statutCirculationOPE',
nullable=True)
if cots_stop_time_status is None:
# if no cots_stop_time_status, it is considered an 'update' of the stop_time
# (can be a delay, back to normal, normal, ...)
cots_base_datetime = _retrieve_stop_event_datetime(
cots_traveler_time)
if cots_base_datetime:
projected_stop_time[
arrival_departure_toggle] = cots_base_datetime
cots_delay = _retrieve_stop_event_delay(
pdp, arrival_departure_toggle)
if cots_delay is not None:
# It's an update only if there is delay
projected_stop_time[
arrival_departure_toggle] += cots_delay
setattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.update.name)
setattr(st_update,
_delay_map[arrival_departure_toggle],
cots_delay)
# otherwise nothing to do (status none, delay none, time none)
elif cots_stop_time_status == 'SUPPRESSION':
# partial delete
setattr(st_update, _status_map[arrival_departure_toggle],
ModificationType.delete.name)
elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT':
# stop_time is replaced by another one
setattr(st_update, _status_map[arrival_departure_toggle],
ModificationType.deleted_for_detour.name)
elif cots_stop_time_status in ['CREATION', 'DETOURNEMENT']:
# new stop_time added
cots_base_datetime = _retrieve_stop_event_datetime(
cots_traveler_time)
if cots_base_datetime:
projected_stop_time[
arrival_departure_toggle] = cots_base_datetime
cots_delay = _retrieve_stop_event_delay(
pdp, arrival_departure_toggle)
if cots_delay is not None:
projected_stop_time[
arrival_departure_toggle] += cots_delay
setattr(st_update,
_stop_event_datetime_map[arrival_departure_toggle],
projected_stop_time[arrival_departure_toggle])
# delay already added to stop_event datetime
setattr(st_update, _delay_map[arrival_departure_toggle],
None)
if cots_stop_time_status == 'CREATION':
# pure add
setattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.add.name)
elif cots_stop_time_status == 'DETOURNEMENT':
# add to replace another stop_time
setattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.added_for_detour.name)
else:
raise InvalidArguments(
'invalid value {} for field horaireVoyageur{}/statutCirculationOPE'
.format(cots_stop_time_status,
arrival_departure_toggle))
arr_dep_status = getattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.none.name)
highest_st_status = get_higher_status(highest_st_status,
arr_dep_status)
self._check_stop_time_consistency(
last_stop_time_depart,
projected_stop_time,
pdp_code='-'.join(pdp[key] for key in ['cr', 'ci', 'ch']))
last_stop_time_depart = projected_stop_time['Depart']
# Calculates effect from stop_time status list(this work is also done in kraken and has to be deleted)
if trip_update.effect == TripEffect.MODIFIED_SERVICE.name:
trip_update.effect = get_effect_by_stop_time_status(
highest_st_status)
return trip_update
def _make_trip_update(self, vj, xml_modification):
"""
create the TripUpdate object
"""
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
delay = xml_modification.find('HoraireProjete')
if delay:
trip_update.status = 'update'
for downstream_point in delay.iter('PointAval'):
# we need only to consider the station
if not as_bool(get_value(downstream_point,
'IndicateurPRGare')):
continue
nav_st, log_dict = self._get_navitia_stop_time(
downstream_point, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict['log'],
contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology',
extra=log_dict)
if nav_st is None:
continue
nav_stop = nav_st.get('stop_point', {})
dep_delay, dep_status = self._get_delay(
downstream_point.find('TypeHoraire/Depart'))
arr_delay, arr_status = self._get_delay(
downstream_point.find('TypeHoraire/Arrivee'))
message = get_value(downstream_point,
'MotifExterne',
nullabe=True)
st_update = model.StopTimeUpdate(nav_stop,
departure_delay=dep_delay,
arrival_delay=arr_delay,
dep_status=dep_status,
arr_status=arr_status,
message=message)
trip_update.stop_time_updates.append(st_update)
removal = xml_modification.find('Suppression')
if removal:
xml_prdebut = removal.find('PRDebut')
if get_value(removal, 'TypeSuppression') == 'T':
trip_update.status = 'delete'
trip_update.stop_time_updates = []
elif get_value(removal, 'TypeSuppression') == 'P':
# it's a partial delete
trip_update.status = 'update'
deleted_points = itertools.chain([removal.find('PRDebut')],
removal.iter('PointSupprime'),
[removal.find('PRFin')])
for deleted_point in deleted_points:
# we need only to consider the stations
if not as_bool(get_value(deleted_point,
'IndicateurPRGare')):
continue
nav_st, log_dict = self._get_navitia_stop_time(
deleted_point, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict['log'],
contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology',
extra=log_dict)
if nav_st is None:
continue
nav_stop = nav_st.get('stop_point', {})
# if the <Depart>/<Arrivee> tags are there, the departure/arrival has been deleted
# regardless of the <Etat> tag
dep_deleted = deleted_point.find(
'TypeHoraire/Depart') is not None
arr_deleted = deleted_point.find(
'TypeHoraire/Arrivee') is not None
dep_status = 'delete' if dep_deleted else 'none'
arr_status = 'delete' if arr_deleted else 'none'
message = get_value(deleted_point,
'MotifExterne',
nullabe=True)
st_update = model.StopTimeUpdate(nav_stop,
dep_status=dep_status,
arr_status=arr_status,
message=message)
trip_update.stop_time_updates.append(st_update)
if xml_prdebut:
trip_update.message = get_value(xml_prdebut,
'MotifExterne',
nullabe=True)
return trip_update
def _get_navitia_vjs(self,
headsign_str,
utc_since_dt,
utc_until_dt,
action_on_trip=ActionOnTrip.NOT_ADDED.name):
"""
Search for navitia's vehicle journeys with given headsigns, in the period provided
:param utc_since_dt: UTC datetime that starts the search period.
Typically the supposed datetime of first base-schedule stop_time.
:param utc_until_dt: UTC datetime that ends the search period.
Typically the supposed datetime of last base-schedule stop_time.
"""
log = logging.getLogger(__name__)
if (utc_since_dt is None) or (utc_until_dt is None):
return []
vjs = {}
# to get the date of the vj we use the start/end of the vj + some tolerance
# since the SNCF data and navitia data might not be synchronized
extended_since_dt = utc_since_dt - SNCF_SEARCH_MARGIN
extended_until_dt = utc_until_dt + SNCF_SEARCH_MARGIN
# using a set to deduplicate
# one headsign_str (ex: "96320/1") can lead to multiple headsigns (ex: ["96320", "96321"])
# but most of the time (if not always) they refer to the same VJ
# (the VJ switches headsign along the way).
# So we do one VJ search for each headsign to ensure we get it, then deduplicate VJs
for train_number in headsigns(headsign_str):
log.debug('searching for vj {} during period [{} - {}] in navitia'.
format(train_number, extended_since_dt,
extended_until_dt))
navitia_vjs = self.navitia.vehicle_journeys(
q={
'headsign': train_number,
'since': to_navitia_str(extended_since_dt),
'until': to_navitia_str(extended_until_dt),
'depth':
'2', # we need this depth to get the stoptime's stop_area
'show_codes':
'true' # we need the stop_points CRCICH codes
})
if action_on_trip == ActionOnTrip.NOT_ADDED.name:
if not navitia_vjs:
logging.getLogger(__name__).info(
'impossible to find train {t} on [{s}, {u}['.format(
t=train_number,
s=extended_since_dt,
u=extended_until_dt))
record_internal_failure('missing train',
contributor=self.contributor)
else:
if action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name and navitia_vjs:
raise InvalidArguments(
'Invalid action, trip {} already present in navitia'.
format(train_number))
navitia_vjs = [make_navitia_empty_vj(train_number)]
for nav_vj in navitia_vjs:
try:
vj = model.VehicleJourney(nav_vj,
extended_since_dt,
extended_until_dt,
vj_start_dt=utc_since_dt)
vjs[nav_vj['id']] = vj
except Exception as e:
logging.getLogger(__name__).exception(
'Error while creating kirin VJ of {}: {}'.format(
nav_vj.get('id'), e))
record_internal_failure('Error while creating kirin VJ',
contributor=self.contributor)
if not vjs:
raise ObjectNotFound(
'no train found for headsign(s) {}'.format(headsign_str))
return vjs.values()
