def build(self, rt_update): """ parse the COTS raw json stored in the rt_update object (in Kirin db) and return a list of trip updates The TripUpdates are not yet associated with the RealTimeUpdate Most of the realtime information parsed is contained in the 'nouvelleVersion' sub-object (see fixtures and documentation) """ try: json = ujson.loads(rt_update.raw_data) except ValueError as e: raise InvalidArguments("invalid json: {}".format(e.message)) if 'nouvelleVersion' not in json: raise InvalidArguments( 'No object "nouvelleVersion" available in feed provided') dict_version = get_value(json, 'nouvelleVersion') vjs = self._get_vjs(dict_version) trip_updates = [self._make_trip_update(vj, dict_version) for vj in vjs] return trip_updates
def post(self, contributor_id): if contributor_id is None or not contributor_id.strip(): abort(400, message="Contributor's id is missing") contributor = Contributor.query_existing().filter_by(id=contributor_id).first() if not contributor: abort(404, message="Contributor '{}' not found".format(contributor_id)) args = flask.globals.request.args data = flask.globals.request.data if not data: raise InvalidArguments("no data provided") source_format = None if "data_format" in args: source_format = args["data_format"].strip().lower() connector_type = ConnectorType(contributor.connector_type) builder_cls = builder_cls_per_connector_type[connector_type] try: data = builder_cls.convert_feed(data, source_format) except Exception as e: raise InvalidArguments(str(e)) wrap_build(builder_cls(contributor), data) return {"message": "'{}' feed processed".format(connector_type.value)}, 200
def _retrieve_projected_time(source_ref, list_proj_time): """ pick the good projected arrival/departure objects from the list provided, using the source-reference if existing """ # if a source-reference is defined if source_ref is not None: # retrieve the one mentioned if it exists if list_proj_time: for p in list_proj_time: s = get_value(p, 'source', nullable=True) if s is not None and s == source_ref: return p # if a reference is provided but impossible to retrieve corresponding element, reject whole COTS feed raise InvalidArguments('invalid json, impossible to find source "{s}" in any json dict ' 'of list: {l}'.format(s=source_ref, l=ujson.dumps(list_proj_time))) elif list_proj_time: # if no source-reference exists, but only one element in the list, we take it if len(list_proj_time) == 1: return list_proj_time[0] # if list has multiple elements but no source-reference, reject whole COTS feed raise InvalidArguments('invalid json, impossible no source but multiple json dicts ' 'in list: {l}'.format(l=ujson.dumps(list_proj_time))) return None
def build(self, rt_update): """ parse raw xml in the rt_update object and return a list of trip updates The TripUpdates are not yet associated with the RealTimeUpdate """ try: root = ElementTree.fromstring(rt_update.raw_data) except ElementTree.ParseError as e: raise InvalidArguments("invalid xml: {}".format(e.message)) if root.tag != 'InfoRetard': raise InvalidArguments( '{} is not a valid xml root, it must be "InfoRetard"'.format( root.tag)) vjs = self._get_vjs(get_node(root, 'Train')) # TODO handle also root[DernierPointDeParcoursObserve] in the modification trip_updates = [ self._make_trip_update(vj, get_node(root, 'TypeModification')) for vj in vjs ] return trip_updates
def str_to_utc_naive_dt(str_time: str) -> datetime: try: dt = parser.parse(str_time, dayfirst=False, yearfirst=True, ignoretz=False) if dt.tzinfo is None: raise InvalidArguments("Datetime must have a timezone set up") return dt.astimezone(utc).replace(tzinfo=None) except Exception as e: raise InvalidArguments( 'Impossible to parse timezoned datetime from "{s}": {m}'.format(s=str_time, m=str(e)) )
def _get_action_on_trip(train_numbers, dict_version, pdps): """ Verify if trip in cots feed is a newly added one and check possible actions :param dict_version: Value of attribut nouvelleVersion in cots feed :return: NOT_ADDED if the trip is not added by current feed, FIRST_TIME_ADDED if it's the first time this trip is added, PREVIOUSLY_ADDED if this train is already added (present in db) """ cots_trip_status = get_value(dict_version, "statutOperationnel", TripStatus.PERTURBEE.name) # We have to verify if the trip exists in database vj_start = _get_first_stop_datetime(pdps, "horaireVoyageurDepart", skip_fully_added_stops=False) vj_end = _get_first_stop_datetime(reversed(pdps), "horaireVoyageurArrivee", skip_fully_added_stops=False) train_id = TRAIN_ID_FORMAT.format(train_numbers) trip_added_in_db = model.TripUpdate.find_vj_by_period( train_id, start_date=vj_start - SNCF_SEARCH_MARGIN, end_date=vj_end + SNCF_SEARCH_MARGIN) action_on_trip = ActionOnTrip.NOT_ADDED.name if trip_added_in_db: # Raise exception on forbidden / inconsistent actions # No addition multiple times # No update or delete on trip already deleted. if cots_trip_status == TripStatus.AJOUTEE.name and trip_added_in_db.status == ModificationType.add.name: raise InvalidArguments( "Invalid action, trip {} can not be added multiple times". format(train_numbers)) elif (cots_trip_status != TripStatus.AJOUTEE.name and trip_added_in_db.status == ModificationType.delete.name): raise InvalidArguments( "Invalid action, trip {} already deleted in database".format( train_numbers)) # Trip added then deleted followed by add should be handled as FIRST_TIME_ADDED if (cots_trip_status == TripStatus.AJOUTEE.name and trip_added_in_db.status == ModificationType.delete.name): action_on_trip = ActionOnTrip.FIRST_TIME_ADDED.name # Trip already added should be handled as PREVIOUSLY_ADDED elif trip_added_in_db.status == ModificationType.add.name: action_on_trip = ActionOnTrip.PREVIOUSLY_ADDED.name else: if cots_trip_status == TripStatus.AJOUTEE.name: action_on_trip = ActionOnTrip.FIRST_TIME_ADDED.name return action_on_trip
def build_trip_updates(self, rt_update): log_dict = {} feed = None try: feed = ElementTree.fromstring(rt_update.raw_data) except ParseError as e: raise InvalidArguments("invalid XML: {}".format(e)) assert isinstance(feed, xml.etree.ElementTree.Element) input_timestamp = self._get_input_timestamp(feed) if input_timestamp is None: self.log.info( "Missing 'ResponseTimestamp' in the SIRI-ET-XML-TN feed; the feed will not be processed" ) return [], log_dict log_dict.update({"input_timestamp": input_timestamp}) self.log.debug( "Start processing siri-et-xml-tn: timestamp = {} ".format( input_timestamp)) tu = self._build_trip_update(feed) if tu is None: msg = "No information for this siri-et-xml-tn with timestamp: {}".format( input_timestamp) set_rtu_status_ko(rt_update, msg, is_reprocess_same_data_allowed=False) self.log.warning(msg) return [], log_dict return [tu], log_dict
def get_ire(req): """ get IRE stream, for the moment, it's the raw xml """ if not req.data: raise InvalidArguments('no ire data provided') return req.data
def get_cots(req): """ get COTS stream, for the moment, it's the raw json """ if not req.data: raise InvalidArguments('no COTS data provided') return req.data
def as_utc_naive_dt(str_time): try: return (parser.parse( str_time, dayfirst=False, yearfirst=True, ignoretz=False).astimezone(utc).replace(tzinfo=None)) except Exception as e: raise InvalidArguments( 'Impossible to parse timezoned datetime from "{s}": {m}'.format( s=str_time, m=e.message))
def _fill_and_check_one_stop_event(st_update, event_toggle, gtfsrt_stop, previous_rt_stop_event_time): """ Fill considered stop-event in given STU, and check its consistency on the fly :param st_update: STU to fill :param event_toggle: stop-event to be considered for filling :param gtfsrt_stop: GTFSRT stop to process :param previous_rt_stop_event_time: datetime of previous stop-event (to check stop-event are well time-sorted) :return: datetime of stop-event (or previous one if none is provided) """ if gtfsrt_stop.HasField("schedule_relationship"): setattr( st_update, STATUS_MAP[event_toggle], stop_gtfsrt_status_to_kirin_status.get(gtfsrt_stop.schedule_relationship).name, ) if gtfsrt_stop.schedule_relationship == gtfs_realtime_pb2.TripUpdate.StopTimeUpdate.NO_DATA: if gtfsrt_stop.HasField(event_toggle): raise InvalidArguments( "invalid feed: stop_point's({}) status is NO_DATA, but '{}' field is provided".format( st_update.id, event_toggle ) ) # if NO_DATA set delay to 0 setattr(st_update, DELAY_MAP[event_toggle], as_duration(0)) if not gtfsrt_stop.HasField(event_toggle): return previous_rt_stop_event_time event = getattr(gtfsrt_stop, event_toggle) if event.HasField("delay"): setattr(st_update, DELAY_MAP[event_toggle], as_duration(event.delay)) if event.HasField("time"): setattr(st_update, event_toggle, timestamp_to_utc_naive_dt(event.time)) if _is_stop_event_served(getattr(st_update, event_toggle), getattr(st_update, STATUS_MAP[event_toggle])): if previous_rt_stop_event_time > getattr(st_update, event_toggle): raise InvalidArguments( "invalid feed: stop_point's({}) time is not consistent".format(st_update.id) ) previous_rt_stop_event_time = getattr(st_update, event_toggle) return previous_rt_stop_event_time
def get_value(sub_json, key, nullable=False): """ get a unique element in an json dict raise an exception if the element does not exists """ res = sub_json.get(key) if res is None and not nullable: raise InvalidArguments('invalid json, impossible to find "{key}" in json dict {elt}'.format( key=key, elt=ujson.dumps(sub_json))) return res
def get_node(elt, xpath, nullabe=False): """ get a unique element in an xml node raise an exception if the element does not exists """ res = elt.find(xpath) if res is None and not nullabe: raise InvalidArguments('invalid xml, impossible to find "{node}" in xml elt {elt}'.format( node=xpath, elt=elt.tag)) return res
def build_trip_updates(self, rt_update): """ parse the COTS raw json stored in the rt_update object (in Kirin db) and return a list of trip updates The TripUpdates are not yet associated with the RealTimeUpdate Most of the realtime information parsed is contained in the 'nouvelleVersion' sub-object (see fixtures and documentation) """ # assuming UTF-8 encoding for all input rt_update.raw_data = rt_update.raw_data.encode("utf-8") try: json = ujson.loads(rt_update.raw_data) except ValueError as e: raise InvalidArguments("invalid json: {}".format(e.message)) if "nouvelleVersion" not in json: raise InvalidArguments( 'No object "nouvelleVersion" available in feed provided') dict_version = get_value(json, "nouvelleVersion") train_numbers = get_value(dict_version, "numeroCourse") pdps = _retrieve_interesting_pdp( get_value(dict_version, "listePointDeParcours")) if not pdps: raise InvalidArguments( 'invalid json, "listePointDeParcours" has no valid stop_time in ' "json elt {elt}".format(elt=ujson.dumps(dict_version))) action_on_trip = _get_action_on_trip(train_numbers, dict_version, pdps) vjs = self._get_vjs(train_numbers, pdps, action_on_trip=action_on_trip) trip_updates = [ self._make_trip_update(dict_version, vj, action_on_trip=action_on_trip) for vj in vjs ] log_dict = {} return trip_updates, log_dict
def _get_navitia_vj(self, piv_key, train_date, ads, is_trip_addition): self.log.debug("searching for vj {} in navitia".format(piv_key)) # large filter on date mostly to ensure only base-schedule VJ are requested filter = 'vehicle_journey.has_code("rt_piv", "{}")'.format(piv_key) train_datetime = datetime.datetime.combine(train_date, datetime.time(0, 0, 0)) since_dt = train_datetime - datetime.timedelta(days=1) until_dt = train_datetime + datetime.timedelta(days=2) navitia_vjs = self._request_navitia_vehicle_journeys(filter, since_dt, until_dt, depth=2, show_codes=True) if not navitia_vjs: # Last PIV information is always right, so if the VJ doesn't exist, it's an ADD (no matter feed content) navitia_vjs = [_make_navitia_empty_vj(piv_key)] vj = None if len(navitia_vjs) != 1: self.log.info( "Can not match a unique train for key {}".format(piv_key)) record_internal_failure("no unique train", contributor=self.contributor.id) else: navitia_vj = navitia_vjs[0] try: base_vs_rt_error_margin = datetime.timedelta(hours=1) vj_base_start = _get_first_stop_base_datetime( ads, "depart", skip_fully_added_stops=not is_trip_addition) # not even a single stop is a base-schedule stop if not vj_base_start: raise InvalidArguments( "Whole trip is specified as pre-existing, but no stop is specified as pre-existing" ) vj = model.VehicleJourney( navitia_vj, vj_base_start - base_vs_rt_error_margin, vj_base_start + base_vs_rt_error_margin, vj_start_dt=vj_base_start, ) except InvalidArguments as i: raise i except Exception as e: self.log.exception( "Error while creating kirin VJ of {}: {}".format( navitia_vjs[0].get("id"), e)) record_internal_failure("Error while creating kirin VJ", contributor=self.contributor.id) if not vj: raise ObjectNotFound("no train found for key {}".format(piv_key)) return vj
def build_trip_updates(self, rt_update): """ parse the gtfs-rt protobuf stored in the rt_update object (in Kirin db) and return a list of trip updates The TripUpdates are not associated with the RealTimeUpdate at this point """ log_dict = {} if not hasattr(rt_update, "proto"): # GOTCHA: should match error message from build_rt_update(), as it will override it. # It allows manage_db_error() to work as expected. # TODO: improve that, but things are quite intricate for error/log management here. raise InvalidArguments("invalid protobuf") proto = rt_update.proto gtfsrt_data_time = timestamp_to_utc_naive_dt(proto.header.timestamp) log_dict.update({"input_timestamp": gtfsrt_data_time}) self.log.debug( "Start processing GTFS-rt: timestamp = {} ({})".format(proto.header.timestamp, gtfsrt_data_time) ) trip_updates = [] for entity in proto.entity: if not entity.HasField("trip_update"): # TODO: log and count in NR continue try: tu = self._build_one_trip_update(entity.trip_update, gtfsrt_data_time=gtfsrt_data_time) if tu is not None: trip_updates.append(tu) except Exception as e: if is_only_warning_exception(e): # TODO: log and count in NR pass else: # TODO: log and count in NR? # we want to track unexpected errors as usual # (and interrupting the feed-processing is OK in that case) raise if not trip_updates: msg = "No information for this gtfs-rt with timestamp: {}".format(proto.header.timestamp) set_rtu_status_ko(rt_update, msg, is_reprocess_same_data_allowed=False) self.log.warning(msg) return trip_updates, log_dict
def _get_vjs(self, json_train): train_numbers = get_value(json_train, 'numeroCourse') pdps = _retrieve_interesting_pdp(get_value(json_train, 'listePointDeParcours')) if not pdps: raise InvalidArguments('invalid json, "listePointDeParcours" has no valid stop_time in ' 'json elt {elt}'.format(elt=ujson.dumps(json_train))) str_time_start = get_value(get_value(pdps[0], 'horaireVoyageurDepart'), 'dateHeure') vj_start = parser.parse(str_time_start, dayfirst=False, yearfirst=True, ignoretz=False) str_time_end = get_value(get_value(pdps[-1], 'horaireVoyageurArrivee'), 'dateHeure') vj_end = parser.parse(str_time_end, dayfirst=False, yearfirst=True, ignoretz=False) return self._get_navitia_vjs(train_numbers, vj_start, vj_end)
def post(self): raw_proto = _get_gtfs_rt(flask.globals.request) from kirin import gtfs_realtime_pb2 # create a raw gtfs-rt obj, save the raw protobuf into the db proto = gtfs_realtime_pb2.FeedMessage() try: proto.ParseFromString(raw_proto) except DecodeError: raise InvalidArguments('invalid protobuf') proto.ParseFromString(raw_proto) model_maker.handle(proto, self.navitia_wrapper, self.contributor) return 'OK', 200
def _check_stop_time_consistency(last_stop_time_depart, projected_stop_time, pdp_code): last_stop_time_depart = last_stop_time_depart if last_stop_time_depart is not None else \ datetime.fromtimestamp(0, tz.tzutc()) projected_arrival = projected_stop_time.get('Arrivee') projected_arrival = projected_arrival if projected_arrival is not None else last_stop_time_depart projected_departure = projected_stop_time.get('Depart') projected_departure = projected_departure if projected_departure is not None else projected_arrival if not (projected_departure >= projected_arrival >= last_stop_time_depart): raise InvalidArguments( 'invalid cots: stop_point\'s({}) time is not consistent'. format(pdp_code))
def _check_stop_time_consistency(last_stop_time_depart, projected_stop_time, pdp_code): last_stop_time_depart = (last_stop_time_depart if last_stop_time_depart is not None else datetime.utcfromtimestamp(0)) projected_arrival = projected_stop_time.get("Arrivee") projected_arrival = projected_arrival if projected_arrival is not None else last_stop_time_depart projected_departure = projected_stop_time.get("Depart") projected_departure = projected_departure if projected_departure is not None else projected_arrival if not (projected_departure >= projected_arrival >= last_stop_time_depart): raise InvalidArguments( "invalid cots: stop_point's({}) time is not consistent".format( pdp_code))
def build_trip_updates(self, rt_update): """ parse the gtfs-rt protobuf stored in the rt_update object (in Kirin db) and return a list of trip updates The TripUpdates are not associated with the RealTimeUpdate at this point """ log_dict = {} if not hasattr(rt_update, "proto"): # GOTCHA: should match error message from build_rt_update(), as it will override it. # It allows manage_db_error() to work as expected. # TODO: improve that, but things are quite intricate for error/log management here. raise InvalidArguments("invalid protobuf") proto = rt_update.proto input_data_time = datetime.datetime.utcfromtimestamp( proto.header.timestamp) log_dict.update({"input_timestamp": input_data_time}) self.log.debug("Start processing GTFS-rt: timestamp = {} ({})".format( proto.header.timestamp, input_data_time)) trip_updates = [] for entity in proto.entity: if not entity.trip_update: continue tu = self._make_trip_updates(entity.trip_update, input_data_time=input_data_time) trip_updates.extend(tu) if not trip_updates: msg = "No information for this gtfs-rt with timestamp: {}".format( proto.header.timestamp) set_rtu_status_ko(rt_update, msg, is_reprocess_same_data_allowed=False) self.log.warning(msg) log_dict = {} return trip_updates, log_dict
def post(self): raw_proto = _get_gtfs_rt(flask.globals.request) from kirin import gtfs_realtime_pb2 # create a raw gtfs-rt obj, save the raw protobuf into the db proto = gtfs_realtime_pb2.FeedMessage() try: proto.ParseFromString(raw_proto) except DecodeError: #We save the non-decodable flux gtfs-rt manage_db_error(proto, 'gtfs-rt', contributor=self.contributor, status='KO', error='Decode Error') raise InvalidArguments('invalid protobuf') else: model_maker.handle(proto, self.navitia_wrapper, self.contributor) return 'OK', 200
def build_trip_updates(self, rt_update): """ parse the PIV raw json stored in the rt_update object (in Kirin db) and return a list of trip updates The TripUpdates are not associated with the RealTimeUpdate at this point """ # assuming UTF-8 encoding for all input rt_update.raw_data = rt_update.raw_data.encode("utf-8") try: json = ujson.loads(rt_update.raw_data) except ValueError as e: raise InvalidArguments("invalid json: {}".format(e.message)) # TODO: build trip_update from PIV feed trip_updates = [] log_dict = {} return trip_updates, log_dict
def _get_vjs(self, json_train): train_numbers = get_value(json_train, 'numeroCourse') pdps = _retrieve_interesting_pdp( get_value(json_train, 'listePointDeParcours')) if not pdps: raise InvalidArguments( 'invalid json, "listePointDeParcours" has no valid stop_time in ' 'json elt {elt}'.format(elt=ujson.dumps(json_train))) # retrieve base-schedule's first departure and last arrival def get_first_fully_added(list_pdps, hour_obj_name): p = next(p for p in list_pdps if not _is_fully_added_pdp(p)) str_time = get_value(get_value(p, hour_obj_name), 'dateHeure') if p else None return parser.parse( str_time, dayfirst=False, yearfirst=True, ignoretz=False) if str_time else None vj_start = get_first_fully_added(pdps, 'horaireVoyageurDepart') vj_end = get_first_fully_added(reversed(pdps), 'horaireVoyageurArrivee') return self._get_navitia_vjs(train_numbers, vj_start, vj_end)
def _make_trip_update(self, vj, json_train): """ create the new TripUpdate object Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md """ logger = logging.getLogger(__name__) trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor trip_message_id = get_value(json_train, 'idMotifInterneReference', nullable=True) if trip_message_id: trip_update.message = self.message_handler.get_message(index=trip_message_id) trip_status = get_value(json_train, 'statutOperationnel') if trip_status == 'SUPPRIMEE': # the whole trip is deleted trip_update.status = 'delete' trip_update.stop_time_updates = [] return trip_update elif trip_status == 'AJOUTEE': # the trip is created from scratch # not handled yet self._record_and_log(logger, 'nouvelleVersion/statutOperationnel == "AJOUTEE" is not handled (yet)') return trip_update # all other status is considered an 'update' of the trip trip_update.status = 'update' pdps = _retrieve_interesting_pdp(get_value(json_train, 'listePointDeParcours')) # manage realtime information stop_time by stop_time for pdp in pdps: # retrieve navitia's stop_time information corresponding to the current COTS pdp nav_st, log_dict = self._get_navitia_stop_time(pdp, vj.navitia_vj) if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_st is None: continue nav_stop = nav_st.get('stop_point', {}) st_update = model.StopTimeUpdate(nav_stop) trip_update.stop_time_updates.append(st_update) # using the message from departure-time in priority, if absent fallback on arrival-time's message st_message_id = get_value(pdp, 'idMotifInterneDepartReference', nullable=True) if not st_message_id: st_message_id = get_value(pdp, 'idMotifInterneArriveeReference', nullable=True) if st_message_id: st_update.message = self.message_handler.get_message(index=st_message_id) _status_map = {'Arrivee': 'arrival_status', 'Depart': 'departure_status'} _delay_map = {'Arrivee': 'arrival_delay', 'Depart': 'departure_delay'} # compute realtime information and fill st_update for arrival and departure for arrival_departure_toggle in ['Arrivee', 'Depart']: cots_traveler_time = get_value(pdp, 'horaireVoyageur{}'.format(arrival_departure_toggle), nullable=True) if cots_traveler_time is None: continue cots_stop_time_status = get_value(cots_traveler_time, 'statutCirculationOPE', nullable=True) if cots_stop_time_status is None: # if no cots_stop_time_status, it is considered an 'update' of the stop_time # (can be a delay, back to normal, normal, ...) cots_ref_planned = get_value(pdp, 'sourceHoraireProjete{}Reference'.format( arrival_departure_toggle), nullable=True) cots_planned_stop_times = get_value(pdp, 'listeHoraireProjete{}'.format(arrival_departure_toggle), nullable=True) cots_planned_stop_time = _retrieve_projected_time(cots_ref_planned, cots_planned_stop_times) if cots_planned_stop_time is None: continue cots_delay = get_value(cots_planned_stop_time, 'pronosticIV', nullable=True) if cots_delay is None: continue setattr(st_update, _status_map[arrival_departure_toggle], 'update') setattr(st_update, _delay_map[arrival_departure_toggle], as_duration(cots_delay)) elif cots_stop_time_status == 'SUPPRESSION': # partial delete setattr(st_update, _status_map[arrival_departure_toggle], 'delete') elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT': # stop_time is replaced by another one self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == ' '"{}" is not handled completely (yet), only removal' .format(cots_stop_time_status)) setattr(st_update, _status_map[arrival_departure_toggle], 'delete') elif cots_stop_time_status == 'CREATION': # new stop_time added self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == ' '"{}" is not handled (yet)'.format(cots_stop_time_status)) elif cots_stop_time_status == 'DETOURNEMENT': # new stop_time added also? self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == ' '"{}" is not handled (yet)'.format(cots_stop_time_status)) else: raise InvalidArguments('invalid value {} for field horaireVoyageur{}/statutCirculationOPE'. format(cots_stop_time_status, arrival_departure_toggle)) return trip_update
def _get_piv(req): if not req.data: raise InvalidArguments("no piv data provided") return req.data
def _make_trip_update(self, json_train, vj, action_on_trip=ActionOnTrip.NOT_ADDED.name): """ create the new TripUpdate object Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md """ trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor trip_message_id = get_value(json_train, 'idMotifInterneReference', nullable=True) if trip_message_id: trip_update.message = self.message_handler.get_message( index=trip_message_id) cots_company_id = get_value(json_train, 'codeCompagnieTransporteur', nullable=True) or DEFAULT_COMPANY_ID trip_update.company_id = self._get_navitia_company(cots_company_id) trip_status = get_value(json_train, 'statutOperationnel') if trip_status == TripStatus.SUPPRIMEE.name: # the whole trip is deleted trip_update.status = ModificationType.delete.name trip_update.stop_time_updates = [] trip_update.effect = TripEffect.NO_SERVICE.name return trip_update elif action_on_trip != ActionOnTrip.NOT_ADDED.name: # the trip is created from scratch trip_update.effect = TripEffect.ADDITIONAL_SERVICE.name trip_update.status = ModificationType.add.name cots_physical_mode = get_value(json_train, 'indicateurFer', nullable=True) trip_update.physical_mode_id = self._get_navitia_physical_mode( cots_physical_mode) trip_update.headsign = get_value(json_train, 'numeroCourse', nullable=True) # all other status is considered an 'update' of the trip_update and effect is calculated # from stop_time status list. This part is also done in kraken and is to be deleted later # Ordered stop_time status= 'nochange', 'add', 'delete', 'update' # 'nochange' or 'update' -> SIGNIFICANT_DELAYS, add -> MODIFIED_SERVICE, delete = DETOUR else: trip_update.status = ModificationType.update.name trip_update.effect = TripEffect.MODIFIED_SERVICE.name # Initialize stop_time status to nochange highest_st_status = ModificationType.none.name pdps = _retrieve_interesting_pdp( get_value(json_train, 'listePointDeParcours')) # this variable is used to memoize the last stop_time's departure in order to check the stop_time consistency # ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure last_stop_time_depart = None # manage realtime information stop_time by stop_time for pdp in pdps: # retrieve navitia's stop_point corresponding to the current COTS pdp nav_stop, log_dict = self._get_navitia_stop_point( pdp, vj.navitia_vj) projected_stop_time = { 'Arrivee': None, 'Depart': None } # used to check consistency if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_stop is None: continue st_update = model.StopTimeUpdate(nav_stop) trip_update.stop_time_updates.append(st_update) # using the message from departure-time in priority, if absent fallback on arrival-time's message st_message_id = get_value(pdp, 'idMotifInterneDepartReference', nullable=True) if not st_message_id: st_message_id = get_value(pdp, 'idMotifInterneArriveeReference', nullable=True) if st_message_id: st_update.message = self.message_handler.get_message( index=st_message_id) _status_map = { 'Arrivee': 'arrival_status', 'Depart': 'departure_status' } _delay_map = { 'Arrivee': 'arrival_delay', 'Depart': 'departure_delay' } _stop_event_datetime_map = { 'Arrivee': 'arrival', 'Depart': 'departure' } # compute realtime information and fill st_update for arrival and departure for arrival_departure_toggle in ['Arrivee', 'Depart']: cots_traveler_time = get_value( pdp, 'horaireVoyageur{}'.format(arrival_departure_toggle), nullable=True) if cots_traveler_time is None: continue cots_stop_time_status = get_value(cots_traveler_time, 'statutCirculationOPE', nullable=True) if cots_stop_time_status is None: # if no cots_stop_time_status, it is considered an 'update' of the stop_time # (can be a delay, back to normal, normal, ...) cots_base_datetime = _retrieve_stop_event_datetime( cots_traveler_time) if cots_base_datetime: projected_stop_time[ arrival_departure_toggle] = cots_base_datetime cots_delay = _retrieve_stop_event_delay( pdp, arrival_departure_toggle) if cots_delay is not None: # It's an update only if there is delay projected_stop_time[ arrival_departure_toggle] += cots_delay setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.update.name) setattr(st_update, _delay_map[arrival_departure_toggle], cots_delay) # otherwise nothing to do (status none, delay none, time none) elif cots_stop_time_status == 'SUPPRESSION': # partial delete setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.delete.name) elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT': # stop_time is replaced by another one setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.deleted_for_detour.name) elif cots_stop_time_status in ['CREATION', 'DETOURNEMENT']: # new stop_time added cots_base_datetime = _retrieve_stop_event_datetime( cots_traveler_time) if cots_base_datetime: projected_stop_time[ arrival_departure_toggle] = cots_base_datetime cots_delay = _retrieve_stop_event_delay( pdp, arrival_departure_toggle) if cots_delay is not None: projected_stop_time[ arrival_departure_toggle] += cots_delay setattr(st_update, _stop_event_datetime_map[arrival_departure_toggle], projected_stop_time[arrival_departure_toggle]) # delay already added to stop_event datetime setattr(st_update, _delay_map[arrival_departure_toggle], None) if cots_stop_time_status == 'CREATION': # pure add setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.add.name) elif cots_stop_time_status == 'DETOURNEMENT': # add to replace another stop_time setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.added_for_detour.name) else: raise InvalidArguments( 'invalid value {} for field horaireVoyageur{}/statutCirculationOPE' .format(cots_stop_time_status, arrival_departure_toggle)) arr_dep_status = getattr(st_update, _status_map[arrival_departure_toggle], ModificationType.none.name) highest_st_status = get_higher_status(highest_st_status, arr_dep_status) self._check_stop_time_consistency( last_stop_time_depart, projected_stop_time, pdp_code='-'.join(pdp[key] for key in ['cr', 'ci', 'ch'])) last_stop_time_depart = projected_stop_time['Depart'] # Calculates effect from stop_time status list(this work is also done in kraken and has to be deleted) if trip_update.effect == TripEffect.MODIFIED_SERVICE.name: trip_update.effect = get_effect_by_stop_time_status( highest_st_status) return trip_update
def _get_gtfs_rt(req): if not req.data: raise InvalidArguments("no gtfs_rt data provided") return req.data
def _make_trip_update(self, json_train, vj): trip_update = model.TripUpdate(vj=vj, contributor_id=self.contributor.id) trip_update.headsign = json_train.get("numero") company_id = json_train.get("operateur").get("codeOperateur") trip_update.company_id = self._get_navitia_company_id(company_id) physical_mode = json_train.get("modeTransport").get("typeMode") trip_update.physical_mode_id = self._get_navitia_physical_mode_id( physical_mode) trip_status_type = trip_piv_status_to_effect.get( json_train.get("evenement").get("type"), TripEffect.UNDEFINED) trip_update.message = json_train.get("evenement").get("texte") trip_update.effect = trip_status_type.name if trip_status_type == TripEffect.NO_SERVICE: # the whole trip is deleted trip_update.status = ModificationType.delete.name trip_update.stop_time_updates = [] return trip_update elif trip_status_type == TripEffect.ADDITIONAL_SERVICE: trip_update.status = ModificationType.add.name else: trip_update.status = ModificationType.update.name highest_st_status = ModificationType.none.name ads = get_value(get_value(json_train, "listeArretsDesserte"), "arret") # this variable is used to store the last stop_time's departure in order to check the stop_time consistency # ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure previous_rt_stop_event_time = datetime.datetime.utcfromtimestamp(0) for arret in ads: # retrieve navitia's stop_point corresponding to the current PIV ad nav_stop, log_dict = self._get_navitia_stop_point( arret, vj.navitia_vj) if log_dict: record_internal_failure(log_dict["log"], contributor=self.contributor.id) log_dict.update({"contributor": self.contributor.id}) logging.getLogger(__name__).info("metrology", extra=log_dict) if nav_stop is None: continue # simply skip stop_times at unknown stop areas st_update = model.StopTimeUpdate(nav_stop) trip_update.stop_time_updates.append(st_update) st_update.message = _get_message(arret) for event_toggle in ["arrivee", "depart"]: event = get_value(arret, event_toggle, nullable=True) if event is None: continue piv_disruption = event.get("evenement", {}) piv_event_status = event.get("statutModification", piv_disruption.get("type", None)) if not piv_event_status or piv_event_status in MANAGED_STOP_EVENTS: piv_event_datetime = get_value(event, "dateHeureReelle", nullable=True) event_datetime = str_to_utc_naive_dt( piv_event_datetime) if piv_event_datetime else None if event_datetime: setattr(st_update, STOP_EVENT_DATETIME_MAP[event_toggle], event_datetime) if piv_event_status in ["SUPPRESSION_PARTIELLE"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.delete.name) elif piv_event_status in ["SUPPRESSION_DETOURNEMENT"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.deleted_for_detour.name) elif piv_event_status in ["CREATION"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.add.name) elif piv_event_status in ["CREATION_DETOURNEMENT"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.added_for_detour.name) elif trip_status_type == TripEffect.ADDITIONAL_SERVICE: # In this case we want to create schedules taking into account the delay # without adding the delay a second time setattr(st_update, STATUS_MAP[event_toggle], ModificationType.add.name) elif piv_event_status in [ "RETARD_OBSERVE", "RETARD_PROJETE" ]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.update.name) if piv_disruption: piv_event_delay = piv_disruption.get( "retard", {}).get("duree", 0) setattr(st_update, DELAY_MAP[event_toggle], as_duration(piv_event_delay * 60)) # minutes else: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.none.name) # otherwise let those be none else: raise InvalidArguments( "invalid value {s} for field {t}/statutModification or {t}/evenement/type" .format(s=piv_event_status, t=event_toggle)) event_status = getattr(st_update, STATUS_MAP[event_toggle], ModificationType.none.name) highest_st_status = get_higher_status(highest_st_status, event_status) mdi = (arret[STOP_EVENT_MDI_MAP[event_toggle]] if arret.get( STOP_EVENT_MDI_MAP[event_toggle]) else False) if _is_stop_event_served(event_datetime, event_status, mdi): if previous_rt_stop_event_time > event_datetime: raise InvalidArguments( "invalid feed: stop_point's({}) time is not consistent" .format( get_value(get_value(arret, "emplacement"), "code"))) previous_rt_stop_event_time = event_datetime # Calculates effect from stop_time status list (this work is also done in kraken and has to be deleted there) if trip_update.effect == TripEffect.MODIFIED_SERVICE.name: trip_update.effect = get_effect_from_modification_type( highest_st_status) return trip_update
def build_trip_updates(self, rt_update): """ parse the PIV raw json stored in the rt_update object (in Kirin db) and return a list of trip updates The TripUpdates are not associated with the RealTimeUpdate at this point """ log_dict = {} try: json = ujson.loads(rt_update.raw_data) except ValueError as e: raise InvalidArguments("invalid json: {}".format(str(e))) dict_objects = get_value(json, "objects") json_train = get_value( dict_objects[0], "object") # TODO: can we get more than 1 relevant in objects[]? brand_code = json_train.get("marque", {}).get("code") if brand_code in ["TN", "TNRER"]: raise InvalidArguments( '"{}" marque code is not supported'.format(brand_code)) piv_disruptions = get_value(json_train, "evenement", nullable=True) plan_transport_source = get_value(json_train, "planTransportSource", nullable=True) if not piv_disruptions and not plan_transport_source: raise InvalidArguments( 'No object "evenement" or "planTransportSource" available in feed provided' ) higher_trip_disruption = _get_piv_higher_trip_disruption( piv_disruptions, plan_transport_source) json_train["evenement"] = higher_trip_disruption train_date_str = get_value(json_train, "dateCirculation") train_numbers = get_value(json_train, "numero") train_company = get_value(get_value(json_train, "operateur"), "codeOperateur") mode_dict = get_value(json_train, "modeTransport") train_mode = get_value(mode_dict, "codeMode") train_submode = get_value(mode_dict, "codeSousMode") train_typemode = get_value(mode_dict, "typeMode") piv_key = "{d}:{n}:{c}:{m}:{s}:{t}".format(d=train_date_str, n=train_numbers, c=train_company, m=train_mode, s=train_submode, t=train_typemode) list_ads = get_value(json_train, "listeArretsDesserte") ads = _retrieve_interesting_stops(get_value(list_ads, "arret")) if len(ads) < 2: raise InvalidArguments( 'invalid json, "listeArretsDesserte/arret" has less than 2 valid stop_times in ' "json elt {elt}".format(elt=ujson.dumps(json_train))) # replace by cleaned and sorted version of stoptimes list. json_train["listeArretsDesserte"]["arret"] = ads is_trip_addition = higher_trip_disruption.get("type") == "CREATION" train_date = str_to_date(train_date_str) vj = self._get_navitia_vj(piv_key, train_date, ads, is_trip_addition) trip_updates = [self._make_trip_update(json_train, vj)] return trip_updates, log_dict