def build(self, rt_update):
"""
parse the COTS raw json stored in the rt_update object (in Kirin db)
and return a list of trip updates
The TripUpdates are not yet associated with the RealTimeUpdate
Most of the realtime information parsed is contained in the 'nouvelleVersion' sub-object
(see fixtures and documentation)
"""
try:
json = ujson.loads(rt_update.raw_data)
except ValueError as e:
raise InvalidArguments("invalid json: {}".format(e.message))
if 'nouvelleVersion' not in json:
raise InvalidArguments(
'No object "nouvelleVersion" available in feed provided')
dict_version = get_value(json, 'nouvelleVersion')
vjs = self._get_vjs(dict_version)
trip_updates = [self._make_trip_update(vj, dict_version) for vj in vjs]
return trip_updates
def post(self, contributor_id):
if contributor_id is None or not contributor_id.strip():
abort(400, message="Contributor's id is missing")
contributor = Contributor.query_existing().filter_by(id=contributor_id).first()
if not contributor:
abort(404, message="Contributor '{}' not found".format(contributor_id))
args = flask.globals.request.args
data = flask.globals.request.data
if not data:
raise InvalidArguments("no data provided")
source_format = None
if "data_format" in args:
source_format = args["data_format"].strip().lower()
connector_type = ConnectorType(contributor.connector_type)
builder_cls = builder_cls_per_connector_type[connector_type]
try:
data = builder_cls.convert_feed(data, source_format)
except Exception as e:
raise InvalidArguments(str(e))
wrap_build(builder_cls(contributor), data)
return {"message": "'{}' feed processed".format(connector_type.value)}, 200
def _retrieve_projected_time(source_ref, list_proj_time):
"""
pick the good projected arrival/departure objects from the list provided,
using the source-reference if existing
"""
# if a source-reference is defined
if source_ref is not None:
# retrieve the one mentioned if it exists
if list_proj_time:
for p in list_proj_time:
s = get_value(p, 'source', nullable=True)
if s is not None and s == source_ref:
return p
# if a reference is provided but impossible to retrieve corresponding element, reject whole COTS feed
raise InvalidArguments('invalid json, impossible to find source "{s}" in any json dict '
'of list: {l}'.format(s=source_ref, l=ujson.dumps(list_proj_time)))
elif list_proj_time:
# if no source-reference exists, but only one element in the list, we take it
if len(list_proj_time) == 1:
return list_proj_time[0]
# if list has multiple elements but no source-reference, reject whole COTS feed
raise InvalidArguments('invalid json, impossible no source but multiple json dicts '
'in list: {l}'.format(l=ujson.dumps(list_proj_time)))
return None
def build(self, rt_update):
"""
parse raw xml in the rt_update object
and return a list of trip updates
The TripUpdates are not yet associated with the RealTimeUpdate
"""
try:
root = ElementTree.fromstring(rt_update.raw_data)
except ElementTree.ParseError as e:
raise InvalidArguments("invalid xml: {}".format(e.message))
if root.tag != 'InfoRetard':
raise InvalidArguments(
'{} is not a valid xml root, it must be "InfoRetard"'.format(
root.tag))
vjs = self._get_vjs(get_node(root, 'Train'))
# TODO handle also root[DernierPointDeParcoursObserve] in the modification
trip_updates = [
self._make_trip_update(vj, get_node(root, 'TypeModification'))
for vj in vjs
]
return trip_updates
def str_to_utc_naive_dt(str_time: str) -> datetime:
try:
dt = parser.parse(str_time, dayfirst=False, yearfirst=True, ignoretz=False)
if dt.tzinfo is None:
raise InvalidArguments("Datetime must have a timezone set up")
return dt.astimezone(utc).replace(tzinfo=None)
except Exception as e:
raise InvalidArguments(
'Impossible to parse timezoned datetime from "{s}": {m}'.format(s=str_time, m=str(e))
)
def _get_action_on_trip(train_numbers, dict_version, pdps):
"""
Verify if trip in cots feed is a newly added one and check possible actions
:param dict_version: Value of attribut nouvelleVersion in cots feed
:return: NOT_ADDED if the trip is not added by current feed,
FIRST_TIME_ADDED if it's the first time this trip is added,
PREVIOUSLY_ADDED if this train is already added (present in db)
"""
cots_trip_status = get_value(dict_version, "statutOperationnel",
TripStatus.PERTURBEE.name)
# We have to verify if the trip exists in database
vj_start = _get_first_stop_datetime(pdps,
"horaireVoyageurDepart",
skip_fully_added_stops=False)
vj_end = _get_first_stop_datetime(reversed(pdps),
"horaireVoyageurArrivee",
skip_fully_added_stops=False)
train_id = TRAIN_ID_FORMAT.format(train_numbers)
trip_added_in_db = model.TripUpdate.find_vj_by_period(
train_id,
start_date=vj_start - SNCF_SEARCH_MARGIN,
end_date=vj_end + SNCF_SEARCH_MARGIN)
action_on_trip = ActionOnTrip.NOT_ADDED.name
if trip_added_in_db:
# Raise exception on forbidden / inconsistent actions
# No addition multiple times
# No update or delete on trip already deleted.
if cots_trip_status == TripStatus.AJOUTEE.name and trip_added_in_db.status == ModificationType.add.name:
raise InvalidArguments(
"Invalid action, trip {} can not be added multiple times".
format(train_numbers))
elif (cots_trip_status != TripStatus.AJOUTEE.name
and trip_added_in_db.status == ModificationType.delete.name):
raise InvalidArguments(
"Invalid action, trip {} already deleted in database".format(
train_numbers))
# Trip added then deleted followed by add should be handled as FIRST_TIME_ADDED
if (cots_trip_status == TripStatus.AJOUTEE.name
and trip_added_in_db.status == ModificationType.delete.name):
action_on_trip = ActionOnTrip.FIRST_TIME_ADDED.name
# Trip already added should be handled as PREVIOUSLY_ADDED
elif trip_added_in_db.status == ModificationType.add.name:
action_on_trip = ActionOnTrip.PREVIOUSLY_ADDED.name
else:
if cots_trip_status == TripStatus.AJOUTEE.name:
action_on_trip = ActionOnTrip.FIRST_TIME_ADDED.name
return action_on_trip
def build_trip_updates(self, rt_update):
log_dict = {}
feed = None
try:
feed = ElementTree.fromstring(rt_update.raw_data)
except ParseError as e:
raise InvalidArguments("invalid XML: {}".format(e))
assert isinstance(feed, xml.etree.ElementTree.Element)
input_timestamp = self._get_input_timestamp(feed)
if input_timestamp is None:
self.log.info(
"Missing 'ResponseTimestamp' in the SIRI-ET-XML-TN feed; the feed will not be processed"
)
return [], log_dict
log_dict.update({"input_timestamp": input_timestamp})
self.log.debug(
"Start processing siri-et-xml-tn: timestamp = {} ".format(
input_timestamp))
tu = self._build_trip_update(feed)
if tu is None:
msg = "No information for this siri-et-xml-tn with timestamp: {}".format(
input_timestamp)
set_rtu_status_ko(rt_update,
msg,
is_reprocess_same_data_allowed=False)
self.log.warning(msg)
return [], log_dict
return [tu], log_dict
def get_ire(req):
"""
get IRE stream, for the moment, it's the raw xml
"""
if not req.data:
raise InvalidArguments('no ire data provided')
return req.data
def get_cots(req):
"""
get COTS stream, for the moment, it's the raw json
"""
if not req.data:
raise InvalidArguments('no COTS data provided')
return req.data
def as_utc_naive_dt(str_time):
try:
return (parser.parse(
str_time, dayfirst=False, yearfirst=True,
ignoretz=False).astimezone(utc).replace(tzinfo=None))
except Exception as e:
raise InvalidArguments(
'Impossible to parse timezoned datetime from "{s}": {m}'.format(
s=str_time, m=e.message))
def _fill_and_check_one_stop_event(st_update, event_toggle, gtfsrt_stop, previous_rt_stop_event_time):
"""
Fill considered stop-event in given STU, and check its consistency on the fly
:param st_update: STU to fill
:param event_toggle: stop-event to be considered for filling
:param gtfsrt_stop: GTFSRT stop to process
:param previous_rt_stop_event_time: datetime of previous stop-event (to check stop-event are well time-sorted)
:return: datetime of stop-event (or previous one if none is provided)
"""
if gtfsrt_stop.HasField("schedule_relationship"):
setattr(
st_update,
STATUS_MAP[event_toggle],
stop_gtfsrt_status_to_kirin_status.get(gtfsrt_stop.schedule_relationship).name,
)
if gtfsrt_stop.schedule_relationship == gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.NO_DATA:
if gtfsrt_stop.HasField(event_toggle):
raise InvalidArguments(
"invalid feed: stop_point's({}) status is NO_DATA, but '{}' field is provided".format(
st_update.id, event_toggle
)
)
# if NO_DATA set delay to 0
setattr(st_update, DELAY_MAP[event_toggle], as_duration(0))
if not gtfsrt_stop.HasField(event_toggle):
return previous_rt_stop_event_time
event = getattr(gtfsrt_stop, event_toggle)
if event.HasField("delay"):
setattr(st_update, DELAY_MAP[event_toggle], as_duration(event.delay))
if event.HasField("time"):
setattr(st_update, event_toggle, timestamp_to_utc_naive_dt(event.time))
if _is_stop_event_served(getattr(st_update, event_toggle), getattr(st_update, STATUS_MAP[event_toggle])):
if previous_rt_stop_event_time > getattr(st_update, event_toggle):
raise InvalidArguments(
"invalid feed: stop_point's({}) time is not consistent".format(st_update.id)
)
previous_rt_stop_event_time = getattr(st_update, event_toggle)
return previous_rt_stop_event_time
def get_value(sub_json, key, nullable=False):
"""
get a unique element in an json dict
raise an exception if the element does not exists
"""
res = sub_json.get(key)
if res is None and not nullable:
raise InvalidArguments('invalid json, impossible to find "{key}" in json dict {elt}'.format(
key=key, elt=ujson.dumps(sub_json)))
return res
def get_node(elt, xpath, nullabe=False):
"""
get a unique element in an xml node
raise an exception if the element does not exists
"""
res = elt.find(xpath)
if res is None and not nullabe:
raise InvalidArguments('invalid xml, impossible to find "{node}" in xml elt {elt}'.format(
node=xpath, elt=elt.tag))
return res
def build_trip_updates(self, rt_update):
"""
parse the COTS raw json stored in the rt_update object (in Kirin db)
and return a list of trip updates
The TripUpdates are not yet associated with the RealTimeUpdate
Most of the realtime information parsed is contained in the 'nouvelleVersion' sub-object
(see fixtures and documentation)
"""
# assuming UTF-8 encoding for all input
rt_update.raw_data = rt_update.raw_data.encode("utf-8")
try:
json = ujson.loads(rt_update.raw_data)
except ValueError as e:
raise InvalidArguments("invalid json: {}".format(e.message))
if "nouvelleVersion" not in json:
raise InvalidArguments(
'No object "nouvelleVersion" available in feed provided')
dict_version = get_value(json, "nouvelleVersion")
train_numbers = get_value(dict_version, "numeroCourse")
pdps = _retrieve_interesting_pdp(
get_value(dict_version, "listePointDeParcours"))
if not pdps:
raise InvalidArguments(
'invalid json, "listePointDeParcours" has no valid stop_time in '
"json elt {elt}".format(elt=ujson.dumps(dict_version)))
action_on_trip = _get_action_on_trip(train_numbers, dict_version, pdps)
vjs = self._get_vjs(train_numbers, pdps, action_on_trip=action_on_trip)
trip_updates = [
self._make_trip_update(dict_version,
vj,
action_on_trip=action_on_trip) for vj in vjs
]
log_dict = {}
return trip_updates, log_dict
def _get_navitia_vj(self, piv_key, train_date, ads, is_trip_addition):
self.log.debug("searching for vj {} in navitia".format(piv_key))
# large filter on date mostly to ensure only base-schedule VJ are requested
filter = 'vehicle_journey.has_code("rt_piv", "{}")'.format(piv_key)
train_datetime = datetime.datetime.combine(train_date,
datetime.time(0, 0, 0))
since_dt = train_datetime - datetime.timedelta(days=1)
until_dt = train_datetime + datetime.timedelta(days=2)
navitia_vjs = self._request_navitia_vehicle_journeys(filter,
since_dt,
until_dt,
depth=2,
show_codes=True)
if not navitia_vjs:
# Last PIV information is always right, so if the VJ doesn't exist, it's an ADD (no matter feed content)
navitia_vjs = [_make_navitia_empty_vj(piv_key)]
vj = None
if len(navitia_vjs) != 1:
self.log.info(
"Can not match a unique train for key {}".format(piv_key))
record_internal_failure("no unique train",
contributor=self.contributor.id)
else:
navitia_vj = navitia_vjs[0]
try:
base_vs_rt_error_margin = datetime.timedelta(hours=1)
vj_base_start = _get_first_stop_base_datetime(
ads, "depart", skip_fully_added_stops=not is_trip_addition)
# not even a single stop is a base-schedule stop
if not vj_base_start:
raise InvalidArguments(
"Whole trip is specified as pre-existing, but no stop is specified as pre-existing"
)
vj = model.VehicleJourney(
navitia_vj,
vj_base_start - base_vs_rt_error_margin,
vj_base_start + base_vs_rt_error_margin,
vj_start_dt=vj_base_start,
)
except InvalidArguments as i:
raise i
except Exception as e:
self.log.exception(
"Error while creating kirin VJ of {}: {}".format(
navitia_vjs[0].get("id"), e))
record_internal_failure("Error while creating kirin VJ",
contributor=self.contributor.id)
if not vj:
raise ObjectNotFound("no train found for key {}".format(piv_key))
return vj
def build_trip_updates(self, rt_update):
"""
parse the gtfs-rt protobuf stored in the rt_update object (in Kirin db)
and return a list of trip updates
The TripUpdates are not associated with the RealTimeUpdate at this point
"""
log_dict = {}
if not hasattr(rt_update, "proto"):
# GOTCHA: should match error message from build_rt_update(), as it will override it.
# It allows manage_db_error() to work as expected.
# TODO: improve that, but things are quite intricate for error/log management here.
raise InvalidArguments("invalid protobuf")
proto = rt_update.proto
gtfsrt_data_time = timestamp_to_utc_naive_dt(proto.header.timestamp)
log_dict.update({"input_timestamp": gtfsrt_data_time})
self.log.debug(
"Start processing GTFS-rt: timestamp = {} ({})".format(proto.header.timestamp, gtfsrt_data_time)
)
trip_updates = []
for entity in proto.entity:
if not entity.HasField("trip_update"):
# TODO: log and count in NR
continue
try:
tu = self._build_one_trip_update(entity.trip_update, gtfsrt_data_time=gtfsrt_data_time)
if tu is not None:
trip_updates.append(tu)
except Exception as e:
if is_only_warning_exception(e):
# TODO: log and count in NR
pass
else:
# TODO: log and count in NR?
# we want to track unexpected errors as usual
# (and interrupting the feed-processing is OK in that case)
raise
if not trip_updates:
msg = "No information for this gtfs-rt with timestamp: {}".format(proto.header.timestamp)
set_rtu_status_ko(rt_update, msg, is_reprocess_same_data_allowed=False)
self.log.warning(msg)
return trip_updates, log_dict
def _get_vjs(self, json_train):
train_numbers = get_value(json_train, 'numeroCourse')
pdps = _retrieve_interesting_pdp(get_value(json_train, 'listePointDeParcours'))
if not pdps:
raise InvalidArguments('invalid json, "listePointDeParcours" has no valid stop_time in '
'json elt {elt}'.format(elt=ujson.dumps(json_train)))
str_time_start = get_value(get_value(pdps[0], 'horaireVoyageurDepart'), 'dateHeure')
vj_start = parser.parse(str_time_start, dayfirst=False, yearfirst=True, ignoretz=False)
str_time_end = get_value(get_value(pdps[-1], 'horaireVoyageurArrivee'), 'dateHeure')
vj_end = parser.parse(str_time_end, dayfirst=False, yearfirst=True, ignoretz=False)
return self._get_navitia_vjs(train_numbers, vj_start, vj_end)
def post(self):
raw_proto = _get_gtfs_rt(flask.globals.request)
from kirin import gtfs_realtime_pb2
# create a raw gtfs-rt obj, save the raw protobuf into the db
proto = gtfs_realtime_pb2.FeedMessage()
try:
proto.ParseFromString(raw_proto)
except DecodeError:
raise InvalidArguments('invalid protobuf')
proto.ParseFromString(raw_proto)
model_maker.handle(proto, self.navitia_wrapper, self.contributor)
return 'OK', 200
def _check_stop_time_consistency(last_stop_time_depart,
projected_stop_time, pdp_code):
last_stop_time_depart = last_stop_time_depart if last_stop_time_depart is not None else \
datetime.fromtimestamp(0, tz.tzutc())
projected_arrival = projected_stop_time.get('Arrivee')
projected_arrival = projected_arrival if projected_arrival is not None else last_stop_time_depart
projected_departure = projected_stop_time.get('Depart')
projected_departure = projected_departure if projected_departure is not None else projected_arrival
if not (projected_departure >= projected_arrival >=
last_stop_time_depart):
raise InvalidArguments(
'invalid cots: stop_point\'s({}) time is not consistent'.
format(pdp_code))
def _check_stop_time_consistency(last_stop_time_depart,
projected_stop_time, pdp_code):
last_stop_time_depart = (last_stop_time_depart if last_stop_time_depart
is not None else datetime.utcfromtimestamp(0))
projected_arrival = projected_stop_time.get("Arrivee")
projected_arrival = projected_arrival if projected_arrival is not None else last_stop_time_depart
projected_departure = projected_stop_time.get("Depart")
projected_departure = projected_departure if projected_departure is not None else projected_arrival
if not (projected_departure >= projected_arrival >=
last_stop_time_depart):
raise InvalidArguments(
"invalid cots: stop_point's({}) time is not consistent".format(
pdp_code))
def build_trip_updates(self, rt_update):
"""
parse the gtfs-rt protobuf stored in the rt_update object (in Kirin db)
and return a list of trip updates
The TripUpdates are not associated with the RealTimeUpdate at this point
"""
log_dict = {}
if not hasattr(rt_update, "proto"):
# GOTCHA: should match error message from build_rt_update(), as it will override it.
# It allows manage_db_error() to work as expected.
# TODO: improve that, but things are quite intricate for error/log management here.
raise InvalidArguments("invalid protobuf")
proto = rt_update.proto
input_data_time = datetime.datetime.utcfromtimestamp(
proto.header.timestamp)
log_dict.update({"input_timestamp": input_data_time})
self.log.debug("Start processing GTFS-rt: timestamp = {} ({})".format(
proto.header.timestamp, input_data_time))
trip_updates = []
for entity in proto.entity:
if not entity.trip_update:
continue
tu = self._make_trip_updates(entity.trip_update,
input_data_time=input_data_time)
trip_updates.extend(tu)
if not trip_updates:
msg = "No information for this gtfs-rt with timestamp: {}".format(
proto.header.timestamp)
set_rtu_status_ko(rt_update,
msg,
is_reprocess_same_data_allowed=False)
self.log.warning(msg)
log_dict = {}
return trip_updates, log_dict
def post(self):
raw_proto = _get_gtfs_rt(flask.globals.request)
from kirin import gtfs_realtime_pb2
# create a raw gtfs-rt obj, save the raw protobuf into the db
proto = gtfs_realtime_pb2.FeedMessage()
try:
proto.ParseFromString(raw_proto)
except DecodeError:
#We save the non-decodable flux gtfs-rt
manage_db_error(proto,
'gtfs-rt',
contributor=self.contributor,
status='KO',
error='Decode Error')
raise InvalidArguments('invalid protobuf')
else:
model_maker.handle(proto, self.navitia_wrapper, self.contributor)
return 'OK', 200
def build_trip_updates(self, rt_update):
"""
parse the PIV raw json stored in the rt_update object (in Kirin db)
and return a list of trip updates
The TripUpdates are not associated with the RealTimeUpdate at this point
"""
# assuming UTF-8 encoding for all input
rt_update.raw_data = rt_update.raw_data.encode("utf-8")
try:
json = ujson.loads(rt_update.raw_data)
except ValueError as e:
raise InvalidArguments("invalid json: {}".format(e.message))
# TODO: build trip_update from PIV feed
trip_updates = []
log_dict = {}
return trip_updates, log_dict
def _get_vjs(self, json_train):
train_numbers = get_value(json_train, 'numeroCourse')
pdps = _retrieve_interesting_pdp(
get_value(json_train, 'listePointDeParcours'))
if not pdps:
raise InvalidArguments(
'invalid json, "listePointDeParcours" has no valid stop_time in '
'json elt {elt}'.format(elt=ujson.dumps(json_train)))
# retrieve base-schedule's first departure and last arrival
def get_first_fully_added(list_pdps, hour_obj_name):
p = next(p for p in list_pdps if not _is_fully_added_pdp(p))
str_time = get_value(get_value(p, hour_obj_name),
'dateHeure') if p else None
return parser.parse(
str_time, dayfirst=False, yearfirst=True,
ignoretz=False) if str_time else None
vj_start = get_first_fully_added(pdps, 'horaireVoyageurDepart')
vj_end = get_first_fully_added(reversed(pdps),
'horaireVoyageurArrivee')
return self._get_navitia_vjs(train_numbers, vj_start, vj_end)
def _make_trip_update(self, vj, json_train):
"""
create the new TripUpdate object
Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md
"""
logger = logging.getLogger(__name__)
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
trip_message_id = get_value(json_train, 'idMotifInterneReference', nullable=True)
if trip_message_id:
trip_update.message = self.message_handler.get_message(index=trip_message_id)
trip_status = get_value(json_train, 'statutOperationnel')
if trip_status == 'SUPPRIMEE':
# the whole trip is deleted
trip_update.status = 'delete'
trip_update.stop_time_updates = []
return trip_update
elif trip_status == 'AJOUTEE':
# the trip is created from scratch
# not handled yet
self._record_and_log(logger, 'nouvelleVersion/statutOperationnel == "AJOUTEE" is not handled (yet)')
return trip_update
# all other status is considered an 'update' of the trip
trip_update.status = 'update'
pdps = _retrieve_interesting_pdp(get_value(json_train, 'listePointDeParcours'))
# manage realtime information stop_time by stop_time
for pdp in pdps:
# retrieve navitia's stop_time information corresponding to the current COTS pdp
nav_st, log_dict = self._get_navitia_stop_time(pdp, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict['log'], contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology', extra=log_dict)
if nav_st is None:
continue
nav_stop = nav_st.get('stop_point', {})
st_update = model.StopTimeUpdate(nav_stop)
trip_update.stop_time_updates.append(st_update)
# using the message from departure-time in priority, if absent fallback on arrival-time's message
st_message_id = get_value(pdp, 'idMotifInterneDepartReference', nullable=True)
if not st_message_id:
st_message_id = get_value(pdp, 'idMotifInterneArriveeReference', nullable=True)
if st_message_id:
st_update.message = self.message_handler.get_message(index=st_message_id)
_status_map = {'Arrivee': 'arrival_status', 'Depart': 'departure_status'}
_delay_map = {'Arrivee': 'arrival_delay', 'Depart': 'departure_delay'}
# compute realtime information and fill st_update for arrival and departure
for arrival_departure_toggle in ['Arrivee', 'Depart']:
cots_traveler_time = get_value(pdp,
'horaireVoyageur{}'.format(arrival_departure_toggle),
nullable=True)
if cots_traveler_time is None:
continue
cots_stop_time_status = get_value(cots_traveler_time,
'statutCirculationOPE',
nullable=True)
if cots_stop_time_status is None:
# if no cots_stop_time_status, it is considered an 'update' of the stop_time
# (can be a delay, back to normal, normal, ...)
cots_ref_planned = get_value(pdp,
'sourceHoraireProjete{}Reference'.format(
arrival_departure_toggle),
nullable=True)
cots_planned_stop_times = get_value(pdp,
'listeHoraireProjete{}'.format(arrival_departure_toggle),
nullable=True)
cots_planned_stop_time = _retrieve_projected_time(cots_ref_planned, cots_planned_stop_times)
if cots_planned_stop_time is None:
continue
cots_delay = get_value(cots_planned_stop_time, 'pronosticIV', nullable=True)
if cots_delay is None:
continue
setattr(st_update, _status_map[arrival_departure_toggle], 'update')
setattr(st_update, _delay_map[arrival_departure_toggle], as_duration(cots_delay))
elif cots_stop_time_status == 'SUPPRESSION':
# partial delete
setattr(st_update, _status_map[arrival_departure_toggle], 'delete')
elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT':
# stop_time is replaced by another one
self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == '
'"{}" is not handled completely (yet), only removal'
.format(cots_stop_time_status))
setattr(st_update, _status_map[arrival_departure_toggle], 'delete')
elif cots_stop_time_status == 'CREATION':
# new stop_time added
self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == '
'"{}" is not handled (yet)'.format(cots_stop_time_status))
elif cots_stop_time_status == 'DETOURNEMENT':
# new stop_time added also?
self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == '
'"{}" is not handled (yet)'.format(cots_stop_time_status))
else:
raise InvalidArguments('invalid value {} for field horaireVoyageur{}/statutCirculationOPE'.
format(cots_stop_time_status, arrival_departure_toggle))
return trip_update
def _get_piv(req):
if not req.data:
raise InvalidArguments("no piv data provided")
return req.data
def _make_trip_update(self,
json_train,
vj,
action_on_trip=ActionOnTrip.NOT_ADDED.name):
"""
create the new TripUpdate object
Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md
"""
trip_update = model.TripUpdate(vj=vj)
trip_update.contributor = self.contributor
trip_message_id = get_value(json_train,
'idMotifInterneReference',
nullable=True)
if trip_message_id:
trip_update.message = self.message_handler.get_message(
index=trip_message_id)
cots_company_id = get_value(json_train,
'codeCompagnieTransporteur',
nullable=True) or DEFAULT_COMPANY_ID
trip_update.company_id = self._get_navitia_company(cots_company_id)
trip_status = get_value(json_train, 'statutOperationnel')
if trip_status == TripStatus.SUPPRIMEE.name:
# the whole trip is deleted
trip_update.status = ModificationType.delete.name
trip_update.stop_time_updates = []
trip_update.effect = TripEffect.NO_SERVICE.name
return trip_update
elif action_on_trip != ActionOnTrip.NOT_ADDED.name:
# the trip is created from scratch
trip_update.effect = TripEffect.ADDITIONAL_SERVICE.name
trip_update.status = ModificationType.add.name
cots_physical_mode = get_value(json_train,
'indicateurFer',
nullable=True)
trip_update.physical_mode_id = self._get_navitia_physical_mode(
cots_physical_mode)
trip_update.headsign = get_value(json_train,
'numeroCourse',
nullable=True)
# all other status is considered an 'update' of the trip_update and effect is calculated
# from stop_time status list. This part is also done in kraken and is to be deleted later
# Ordered stop_time status= 'nochange', 'add', 'delete', 'update'
# 'nochange' or 'update' -> SIGNIFICANT_DELAYS, add -> MODIFIED_SERVICE, delete = DETOUR
else:
trip_update.status = ModificationType.update.name
trip_update.effect = TripEffect.MODIFIED_SERVICE.name
# Initialize stop_time status to nochange
highest_st_status = ModificationType.none.name
pdps = _retrieve_interesting_pdp(
get_value(json_train, 'listePointDeParcours'))
# this variable is used to memoize the last stop_time's departure in order to check the stop_time consistency
# ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure
last_stop_time_depart = None
# manage realtime information stop_time by stop_time
for pdp in pdps:
# retrieve navitia's stop_point corresponding to the current COTS pdp
nav_stop, log_dict = self._get_navitia_stop_point(
pdp, vj.navitia_vj)
projected_stop_time = {
'Arrivee': None,
'Depart': None
} # used to check consistency
if log_dict:
record_internal_failure(log_dict['log'],
contributor=self.contributor)
log_dict.update({'contributor': self.contributor})
logging.getLogger(__name__).info('metrology', extra=log_dict)
if nav_stop is None:
continue
st_update = model.StopTimeUpdate(nav_stop)
trip_update.stop_time_updates.append(st_update)
# using the message from departure-time in priority, if absent fallback on arrival-time's message
st_message_id = get_value(pdp,
'idMotifInterneDepartReference',
nullable=True)
if not st_message_id:
st_message_id = get_value(pdp,
'idMotifInterneArriveeReference',
nullable=True)
if st_message_id:
st_update.message = self.message_handler.get_message(
index=st_message_id)
_status_map = {
'Arrivee': 'arrival_status',
'Depart': 'departure_status'
}
_delay_map = {
'Arrivee': 'arrival_delay',
'Depart': 'departure_delay'
}
_stop_event_datetime_map = {
'Arrivee': 'arrival',
'Depart': 'departure'
}
# compute realtime information and fill st_update for arrival and departure
for arrival_departure_toggle in ['Arrivee', 'Depart']:
cots_traveler_time = get_value(
pdp,
'horaireVoyageur{}'.format(arrival_departure_toggle),
nullable=True)
if cots_traveler_time is None:
continue
cots_stop_time_status = get_value(cots_traveler_time,
'statutCirculationOPE',
nullable=True)
if cots_stop_time_status is None:
# if no cots_stop_time_status, it is considered an 'update' of the stop_time
# (can be a delay, back to normal, normal, ...)
cots_base_datetime = _retrieve_stop_event_datetime(
cots_traveler_time)
if cots_base_datetime:
projected_stop_time[
arrival_departure_toggle] = cots_base_datetime
cots_delay = _retrieve_stop_event_delay(
pdp, arrival_departure_toggle)
if cots_delay is not None:
# It's an update only if there is delay
projected_stop_time[
arrival_departure_toggle] += cots_delay
setattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.update.name)
setattr(st_update,
_delay_map[arrival_departure_toggle],
cots_delay)
# otherwise nothing to do (status none, delay none, time none)
elif cots_stop_time_status == 'SUPPRESSION':
# partial delete
setattr(st_update, _status_map[arrival_departure_toggle],
ModificationType.delete.name)
elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT':
# stop_time is replaced by another one
setattr(st_update, _status_map[arrival_departure_toggle],
ModificationType.deleted_for_detour.name)
elif cots_stop_time_status in ['CREATION', 'DETOURNEMENT']:
# new stop_time added
cots_base_datetime = _retrieve_stop_event_datetime(
cots_traveler_time)
if cots_base_datetime:
projected_stop_time[
arrival_departure_toggle] = cots_base_datetime
cots_delay = _retrieve_stop_event_delay(
pdp, arrival_departure_toggle)
if cots_delay is not None:
projected_stop_time[
arrival_departure_toggle] += cots_delay
setattr(st_update,
_stop_event_datetime_map[arrival_departure_toggle],
projected_stop_time[arrival_departure_toggle])
# delay already added to stop_event datetime
setattr(st_update, _delay_map[arrival_departure_toggle],
None)
if cots_stop_time_status == 'CREATION':
# pure add
setattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.add.name)
elif cots_stop_time_status == 'DETOURNEMENT':
# add to replace another stop_time
setattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.added_for_detour.name)
else:
raise InvalidArguments(
'invalid value {} for field horaireVoyageur{}/statutCirculationOPE'
.format(cots_stop_time_status,
arrival_departure_toggle))
arr_dep_status = getattr(st_update,
_status_map[arrival_departure_toggle],
ModificationType.none.name)
highest_st_status = get_higher_status(highest_st_status,
arr_dep_status)
self._check_stop_time_consistency(
last_stop_time_depart,
projected_stop_time,
pdp_code='-'.join(pdp[key] for key in ['cr', 'ci', 'ch']))
last_stop_time_depart = projected_stop_time['Depart']
# Calculates effect from stop_time status list(this work is also done in kraken and has to be deleted)
if trip_update.effect == TripEffect.MODIFIED_SERVICE.name:
trip_update.effect = get_effect_by_stop_time_status(
highest_st_status)
return trip_update
def _get_gtfs_rt(req):
if not req.data:
raise InvalidArguments("no gtfs_rt data provided")
return req.data
def _make_trip_update(self, json_train, vj):
trip_update = model.TripUpdate(vj=vj,
contributor_id=self.contributor.id)
trip_update.headsign = json_train.get("numero")
company_id = json_train.get("operateur").get("codeOperateur")
trip_update.company_id = self._get_navitia_company_id(company_id)
physical_mode = json_train.get("modeTransport").get("typeMode")
trip_update.physical_mode_id = self._get_navitia_physical_mode_id(
physical_mode)
trip_status_type = trip_piv_status_to_effect.get(
json_train.get("evenement").get("type"), TripEffect.UNDEFINED)
trip_update.message = json_train.get("evenement").get("texte")
trip_update.effect = trip_status_type.name
if trip_status_type == TripEffect.NO_SERVICE:
# the whole trip is deleted
trip_update.status = ModificationType.delete.name
trip_update.stop_time_updates = []
return trip_update
elif trip_status_type == TripEffect.ADDITIONAL_SERVICE:
trip_update.status = ModificationType.add.name
else:
trip_update.status = ModificationType.update.name
highest_st_status = ModificationType.none.name
ads = get_value(get_value(json_train, "listeArretsDesserte"), "arret")
# this variable is used to store the last stop_time's departure in order to check the stop_time consistency
# ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure
previous_rt_stop_event_time = datetime.datetime.utcfromtimestamp(0)
for arret in ads:
# retrieve navitia's stop_point corresponding to the current PIV ad
nav_stop, log_dict = self._get_navitia_stop_point(
arret, vj.navitia_vj)
if log_dict:
record_internal_failure(log_dict["log"],
contributor=self.contributor.id)
log_dict.update({"contributor": self.contributor.id})
logging.getLogger(__name__).info("metrology", extra=log_dict)
if nav_stop is None:
continue # simply skip stop_times at unknown stop areas
st_update = model.StopTimeUpdate(nav_stop)
trip_update.stop_time_updates.append(st_update)
st_update.message = _get_message(arret)
for event_toggle in ["arrivee", "depart"]:
event = get_value(arret, event_toggle, nullable=True)
if event is None:
continue
piv_disruption = event.get("evenement", {})
piv_event_status = event.get("statutModification",
piv_disruption.get("type", None))
if not piv_event_status or piv_event_status in MANAGED_STOP_EVENTS:
piv_event_datetime = get_value(event,
"dateHeureReelle",
nullable=True)
event_datetime = str_to_utc_naive_dt(
piv_event_datetime) if piv_event_datetime else None
if event_datetime:
setattr(st_update,
STOP_EVENT_DATETIME_MAP[event_toggle],
event_datetime)
if piv_event_status in ["SUPPRESSION_PARTIELLE"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.delete.name)
elif piv_event_status in ["SUPPRESSION_DETOURNEMENT"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.deleted_for_detour.name)
elif piv_event_status in ["CREATION"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.add.name)
elif piv_event_status in ["CREATION_DETOURNEMENT"]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.added_for_detour.name)
elif trip_status_type == TripEffect.ADDITIONAL_SERVICE:
# In this case we want to create schedules taking into account the delay
# without adding the delay a second time
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.add.name)
elif piv_event_status in [
"RETARD_OBSERVE", "RETARD_PROJETE"
]:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.update.name)
if piv_disruption:
piv_event_delay = piv_disruption.get(
"retard", {}).get("duree", 0)
setattr(st_update, DELAY_MAP[event_toggle],
as_duration(piv_event_delay *
60)) # minutes
else:
setattr(st_update, STATUS_MAP[event_toggle],
ModificationType.none.name)
# otherwise let those be none
else:
raise InvalidArguments(
"invalid value {s} for field {t}/statutModification or {t}/evenement/type"
.format(s=piv_event_status, t=event_toggle))
event_status = getattr(st_update, STATUS_MAP[event_toggle],
ModificationType.none.name)
highest_st_status = get_higher_status(highest_st_status,
event_status)
mdi = (arret[STOP_EVENT_MDI_MAP[event_toggle]] if arret.get(
STOP_EVENT_MDI_MAP[event_toggle]) else False)
if _is_stop_event_served(event_datetime, event_status, mdi):
if previous_rt_stop_event_time > event_datetime:
raise InvalidArguments(
"invalid feed: stop_point's({}) time is not consistent"
.format(
get_value(get_value(arret, "emplacement"),
"code")))
previous_rt_stop_event_time = event_datetime
# Calculates effect from stop_time status list (this work is also done in kraken and has to be deleted there)
if trip_update.effect == TripEffect.MODIFIED_SERVICE.name:
trip_update.effect = get_effect_from_modification_type(
highest_st_status)
return trip_update
def build_trip_updates(self, rt_update):
"""
parse the PIV raw json stored in the rt_update object (in Kirin db)
and return a list of trip updates
The TripUpdates are not associated with the RealTimeUpdate at this point
"""
log_dict = {}
try:
json = ujson.loads(rt_update.raw_data)
except ValueError as e:
raise InvalidArguments("invalid json: {}".format(str(e)))
dict_objects = get_value(json, "objects")
json_train = get_value(
dict_objects[0],
"object") # TODO: can we get more than 1 relevant in objects[]?
brand_code = json_train.get("marque", {}).get("code")
if brand_code in ["TN", "TNRER"]:
raise InvalidArguments(
'"{}" marque code is not supported'.format(brand_code))
piv_disruptions = get_value(json_train, "evenement", nullable=True)
plan_transport_source = get_value(json_train,
"planTransportSource",
nullable=True)
if not piv_disruptions and not plan_transport_source:
raise InvalidArguments(
'No object "evenement" or "planTransportSource" available in feed provided'
)
higher_trip_disruption = _get_piv_higher_trip_disruption(
piv_disruptions, plan_transport_source)
json_train["evenement"] = higher_trip_disruption
train_date_str = get_value(json_train, "dateCirculation")
train_numbers = get_value(json_train, "numero")
train_company = get_value(get_value(json_train, "operateur"),
"codeOperateur")
mode_dict = get_value(json_train, "modeTransport")
train_mode = get_value(mode_dict, "codeMode")
train_submode = get_value(mode_dict, "codeSousMode")
train_typemode = get_value(mode_dict, "typeMode")
piv_key = "{d}:{n}:{c}:{m}:{s}:{t}".format(d=train_date_str,
n=train_numbers,
c=train_company,
m=train_mode,
s=train_submode,
t=train_typemode)
list_ads = get_value(json_train, "listeArretsDesserte")
ads = _retrieve_interesting_stops(get_value(list_ads, "arret"))
if len(ads) < 2:
raise InvalidArguments(
'invalid json, "listeArretsDesserte/arret" has less than 2 valid stop_times in '
"json elt {elt}".format(elt=ujson.dumps(json_train)))
# replace by cleaned and sorted version of stoptimes list.
json_train["listeArretsDesserte"]["arret"] = ads
is_trip_addition = higher_trip_disruption.get("type") == "CREATION"
train_date = str_to_date(train_date_str)
vj = self._get_navitia_vj(piv_key, train_date, ads, is_trip_addition)
trip_updates = [self._make_trip_update(json_train, vj)]
return trip_updates, log_dict
