def setup_database(): """ we create two realtime_updates with the same vj but for different date and return a vj for navitia """ with app.app_context(): vj1 = model.VehicleJourney({'trip': {'id': 'vj:1'}}, date(2015, 11, 4)) vj2 = model.VehicleJourney({'trip': {'id': 'vj:2'}}, date(2015, 11, 4)) vj3 = model.VehicleJourney({'trip': {'id': 'vj:3'}}, date(2015, 11, 4)) tu1 = model.TripUpdate(vj1, contributor='realtime.ire') tu2 = model.TripUpdate(vj2, contributor='realtime.ire') tu3 = model.TripUpdate(vj3, contributor='realtime.timeo') rtu1 = model.RealTimeUpdate(None, 'ire') rtu1.created_at = datetime(2015, 11, 4, 6, 32) rtu1.trip_updates.append(tu1) model.db.session.add(rtu1) rtu2 = model.RealTimeUpdate(None, 'ire') rtu2.created_at = datetime(2015, 11, 4, 7, 32) rtu2.trip_updates.append(tu2) model.db.session.add(rtu2) rtu3 = model.RealTimeUpdate(None, 'ire') rtu3.created_at = datetime(2015, 11, 4, 7, 42) rtu3.trip_updates.append(tu3) model.db.session.add(rtu3) model.db.session.commit()
def _build_one_trip_update(self, gtfsrt_trip_update, gtfsrt_data_time): """ Build the smallest TripUpdate from GTFSRT trip, don't try to fill missing information yet Goal is to just convert information to Kirin's model (and add navitia PT objects information, like navitia ids, etc.) """ if not gtfsrt_trip_update.HasField("trip") or not gtfsrt_trip_update.trip.HasField("trip_id"): # TODO: log and count in NR return None # Processing only scheduled trips currently if ( gtfsrt_trip_update.trip.HasField("schedule_relationship") and gtfsrt_trip_update.trip.schedule_relationship != gtfs_realtime_pb2.TripDescriptor.SCHEDULED ): # TODO: log and count in NR return None since_dt = floor_datetime(gtfsrt_data_time - self.period_filter_tolerance) until_dt = floor_datetime(gtfsrt_data_time + self.period_filter_tolerance + datetime.timedelta(hours=1)) self.log.info("get_nav_vj") vj = self._get_navitia_vj(gtfsrt_trip_update.trip.trip_id, since_dt=since_dt, until_dt=until_dt) if vj is None: # if not able to find the base-schedule trip, reject GTFSRT trip as no ADD is handled # TODO: log and count in NR return None trip_update = model.TripUpdate(vj=vj, contributor_id=self.contributor.id) if gtfsrt_trip_update.trip.HasField("schedule_relationship"): trip_effect = trip_gtfsrt_status_to_effect.get(gtfsrt_trip_update.trip.schedule_relationship) if trip_effect is not None: trip_update.effect = trip_effect.name trip_update.status = get_trip_status_from_effect(trip_update.effect) # this variable is used to store the last stop_time's departure in order to check the stop_time consistency # ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure previous_rt_stop_event_time = timestamp_to_utc_naive_dt(0) for gtfsrt_stop in gtfsrt_trip_update.stop_time_update: try: st_update, last_stop_event_time = self._build_one_stop_time_update( gtfsrt_stop=gtfsrt_stop, nav_vj=vj.navitia_vj, previous_rt_stop_event_time=previous_rt_stop_event_time, ) if None not in [st_update, last_stop_event_time]: trip_update.stop_time_updates.append(st_update) previous_rt_stop_event_time = last_stop_event_time except Exception as _: del trip_update.stop_time_updates[:] raise return trip_update
def _make_trip_update(self, vj, xml_modification): """ create the TripUpdate object """ trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor delay = xml_modification.find('HoraireProjete') if delay: trip_update.status = 'update' for downstream_point in delay.iter('PointAval'): # we need only to consider the station if not as_bool(get_value(downstream_point, 'IndicateurPRGare')): continue nav_st = self._get_navitia_stop_time(downstream_point, vj.navitia_vj) if nav_st is None: continue nav_stop = nav_st.get('stop_point', {}) dep_delay, dep_status = self._get_delay( downstream_point.find('TypeHoraire/Depart')) arr_delay, arr_status = self._get_delay( downstream_point.find('TypeHoraire/Arrivee')) message = get_value(downstream_point, 'MotifExterne') st_update = model.StopTimeUpdate(nav_stop, departure_delay=dep_delay, arrival_delay=arr_delay, dep_status=dep_status, arr_status=arr_status, message=message) trip_update.stop_time_updates.append(st_update) removal = xml_modification.find('Suppression') if removal: if get_value(removal, 'TypeSuppression') == 'T': trip_update.status = 'delete' trip_update.stop_time_updates = [] elif get_value(removal, 'TypeSuppression') == 'P': trip_update.status = 'update' xml_prdebut = removal.find('PRDebut') if xml_prdebut: trip_update.message = get_value(xml_prdebut, 'MotifExterne') return trip_update
def _make_trip_updates(self, input_trip_update, data_time): vjs = self._get_navitia_vjs(input_trip_update.trip, data_time=data_time) trip_updates = [] for vj in vjs: trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor trip_updates.append(trip_update) for input_st_update in input_trip_update.stop_time_update: st_update = self._make_stoptime_update(input_st_update, vj.navitia_vj) if not st_update: continue trip_update.stop_time_updates.append(st_update) return trip_updates
def build_trip_update_if_modified( db_trip_update: Optional[model.TripUpdate], old_stus: List[model.StopTimeUpdate], new_trip_update: model.TripUpdate, new_stus: List[model.StopTimeUpdate], contributor_id: str, ) -> Optional[model.TripUpdate]: """ Build and return a consolidated TripUpdate (with its attached StopTimeUpdates) if modifications appeared in new one :param db_trip_update: TripUpdate obtained from db if previous realtime processing result was stored (or None) :param old_stus: ordered STU list of previous realtime processing result (or corresponding list from base-schedule) :param new_trip_update: TripUpdate from new feed being processed :param new_stus: ordered STU list of new feed being processed :param contributor_id: contributor's id :return: TripUpdate with attached StopTimeUpdates if new is different from old (or from base-schedule), otherwise None """ old_tu = db_trip_update or model.TripUpdate(vj=None, contributor_id=contributor_id) if are_trip_updates_equal( left_tu=old_tu, left_stus=old_stus, right_tu=new_trip_update, right_stus=new_stus, ): return None # Update existing TripUpdate in order to: # * avoid duplicates (unicity constraint) # * keep created_at # * preserve performance (ORM objects are heavy) res = db_trip_update if db_trip_update else new_trip_update res.copy_processed_attributes(new_trip_update) # As SQLAlchemy persists TU in db when STU are attached, # attach StopTimeUpdates only at the end to avoid breaking unicity constraint that could happen # when new_trip_update exists at the same time than the result of a previous realtime processing res.stop_time_updates = new_stus return res
def _make_trip_update(self, vj, xml_modification): """ create the TripUpdate object """ trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor delay = xml_modification.find('HoraireProjete') if delay: trip_update.status = 'update' for downstream_point in delay.iter('PointAval'): # we need only to consider the station if not as_bool(get_value(downstream_point, 'IndicateurPRGare')): continue nav_st, log_dict = self._get_navitia_stop_time( downstream_point, vj.navitia_vj) if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_st is None: continue nav_stop = nav_st.get('stop_point', {}) dep_delay, dep_status = self._get_delay( downstream_point.find('TypeHoraire/Depart')) arr_delay, arr_status = self._get_delay( downstream_point.find('TypeHoraire/Arrivee')) message = get_value(downstream_point, 'MotifExterne', nullabe=True) st_update = model.StopTimeUpdate(nav_stop, departure_delay=dep_delay, arrival_delay=arr_delay, dep_status=dep_status, arr_status=arr_status, message=message) trip_update.stop_time_updates.append(st_update) removal = xml_modification.find('Suppression') if removal: xml_prdebut = removal.find('PRDebut') if get_value(removal, 'TypeSuppression') == 'T': trip_update.status = 'delete' trip_update.stop_time_updates = [] elif get_value(removal, 'TypeSuppression') == 'P': # it's a partial delete trip_update.status = 'update' deleted_points = itertools.chain([removal.find('PRDebut')], removal.iter('PointSupprime'), [removal.find('PRFin')]) for deleted_point in deleted_points: # we need only to consider the stations if not as_bool(get_value(deleted_point, 'IndicateurPRGare')): continue nav_st, log_dict = self._get_navitia_stop_time( deleted_point, vj.navitia_vj) if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_st is None: continue nav_stop = nav_st.get('stop_point', {}) # if the <Depart>/<Arrivee> tags are there, the departure/arrival has been deleted # regardless of the <Etat> tag dep_deleted = deleted_point.find( 'TypeHoraire/Depart') is not None arr_deleted = deleted_point.find( 'TypeHoraire/Arrivee') is not None dep_status = 'delete' if dep_deleted else 'none' arr_status = 'delete' if arr_deleted else 'none' message = get_value(deleted_point, 'MotifExterne', nullabe=True) st_update = model.StopTimeUpdate(nav_stop, dep_status=dep_status, arr_status=arr_status, message=message) trip_update.stop_time_updates.append(st_update) if xml_prdebut: trip_update.message = get_value(xml_prdebut, 'MotifExterne', nullabe=True) return trip_update
def _make_trip_updates(self, input_trip_update, input_data_time): """ If trip_update.stop_time_updates is not a strict ending subset of vj.stop_times we reject the trip update On the other hand: 1. For the stop point present in trip_update.stop_time_updates we create a trip_update merging informations with that of navitia stop 2. For the first stop point absent in trip_update.stop_time_updates we create a stop_time_update with no delay for that stop """ vjs = self._get_navitia_vjs(input_trip_update.trip, input_data_time=input_data_time) trip_updates = [] for vj in vjs: trip_update = model.TripUpdate(vj=vj, contributor_id=self.contributor.id) highest_st_status = ModificationType.none.name is_tu_valid = True vj_stop_order = len(vj.navitia_vj.get("stop_times", [])) - 1 for vj_stop, tu_stop in itertools.zip_longest( reversed(vj.navitia_vj.get("stop_times", [])), reversed(input_trip_update.stop_time_update)): if vj_stop is None: is_tu_valid = False break vj_stop_point = vj_stop.get("stop_point") if vj_stop_point is None: is_tu_valid = False break if tu_stop is not None: if self._get_stop_code(vj_stop_point) != tu_stop.stop_id: is_tu_valid = False break tu_stop.stop_sequence = vj_stop_order st_update = _make_stoptime_update(tu_stop, vj_stop_point) if st_update is not None: trip_update.stop_time_updates.append(st_update) else: # Initialize stops absent in trip_updates but present in vj st_update = _init_stop_update(vj_stop_point, vj_stop_order) if st_update is not None: trip_update.stop_time_updates.append(st_update) for status in [ st_update.departure_status, st_update.arrival_status ]: highest_st_status = get_higher_status( highest_st_status, status) vj_stop_order -= 1 if is_tu_valid: # Since vj.stop_times are managed in reversed order, we re sort stop_time_updates by order. trip_update.stop_time_updates.sort(key=lambda x: x.order) trip_update.effect = get_effect_from_modification_type( highest_st_status) trip_updates.append(trip_update) else: self.log.warning( "stop_time_update do not match with stops in navitia for trip : {} timestamp: {}" .format(input_trip_update.trip.trip_id, calendar.timegm(input_data_time.utctimetuple()))) record_internal_failure( "stop_time_update do not match with stops in navitia", contributor=self.contributor.id) del trip_update.stop_time_updates[:] return trip_updates
def _make_trip_update(self, json_train, vj): trip_update = model.TripUpdate(vj=vj, contributor_id=self.contributor.id) trip_update.headsign = json_train.get("numero") company_id = json_train.get("operateur").get("codeOperateur") trip_update.company_id = self._get_navitia_company_id(company_id) physical_mode = json_train.get("modeTransport").get("typeMode") trip_update.physical_mode_id = self._get_navitia_physical_mode_id( physical_mode) trip_status_type = trip_piv_status_to_effect.get( json_train.get("evenement").get("type"), TripEffect.UNDEFINED) trip_update.message = json_train.get("evenement").get("texte") trip_update.effect = trip_status_type.name if trip_status_type == TripEffect.NO_SERVICE: # the whole trip is deleted trip_update.status = ModificationType.delete.name trip_update.stop_time_updates = [] return trip_update elif trip_status_type == TripEffect.ADDITIONAL_SERVICE: trip_update.status = ModificationType.add.name else: trip_update.status = ModificationType.update.name highest_st_status = ModificationType.none.name ads = get_value(get_value(json_train, "listeArretsDesserte"), "arret") # this variable is used to store the last stop_time's departure in order to check the stop_time consistency # ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure previous_rt_stop_event_time = datetime.datetime.utcfromtimestamp(0) for arret in ads: # retrieve navitia's stop_point corresponding to the current PIV ad nav_stop, log_dict = self._get_navitia_stop_point( arret, vj.navitia_vj) if log_dict: record_internal_failure(log_dict["log"], contributor=self.contributor.id) log_dict.update({"contributor": self.contributor.id}) logging.getLogger(__name__).info("metrology", extra=log_dict) if nav_stop is None: continue # simply skip stop_times at unknown stop areas st_update = model.StopTimeUpdate(nav_stop) trip_update.stop_time_updates.append(st_update) st_update.message = _get_message(arret) for event_toggle in ["arrivee", "depart"]: event = get_value(arret, event_toggle, nullable=True) if event is None: continue piv_disruption = event.get("evenement", {}) piv_event_status = event.get("statutModification", piv_disruption.get("type", None)) if not piv_event_status or piv_event_status in MANAGED_STOP_EVENTS: piv_event_datetime = get_value(event, "dateHeureReelle", nullable=True) event_datetime = str_to_utc_naive_dt( piv_event_datetime) if piv_event_datetime else None if event_datetime: setattr(st_update, STOP_EVENT_DATETIME_MAP[event_toggle], event_datetime) if piv_event_status in ["SUPPRESSION_PARTIELLE"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.delete.name) elif piv_event_status in ["SUPPRESSION_DETOURNEMENT"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.deleted_for_detour.name) elif piv_event_status in ["CREATION"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.add.name) elif piv_event_status in ["CREATION_DETOURNEMENT"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.added_for_detour.name) elif trip_status_type == TripEffect.ADDITIONAL_SERVICE: # In this case we want to create schedules taking into account the delay # without adding the delay a second time setattr(st_update, STATUS_MAP[event_toggle], ModificationType.add.name) elif piv_event_status in [ "RETARD_OBSERVE", "RETARD_PROJETE" ]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.update.name) if piv_disruption: piv_event_delay = piv_disruption.get( "retard", {}).get("duree", 0) setattr(st_update, DELAY_MAP[event_toggle], as_duration(piv_event_delay * 60)) # minutes else: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.none.name) # otherwise let those be none else: raise InvalidArguments( "invalid value {s} for field {t}/statutModification or {t}/evenement/type" .format(s=piv_event_status, t=event_toggle)) event_status = getattr(st_update, STATUS_MAP[event_toggle], ModificationType.none.name) highest_st_status = get_higher_status(highest_st_status, event_status) mdi = (arret[STOP_EVENT_MDI_MAP[event_toggle]] if arret.get( STOP_EVENT_MDI_MAP[event_toggle]) else False) if _is_stop_event_served(event_datetime, event_status, mdi): if previous_rt_stop_event_time > event_datetime: raise InvalidArguments( "invalid feed: stop_point's({}) time is not consistent" .format( get_value(get_value(arret, "emplacement"), "code"))) previous_rt_stop_event_time = event_datetime # Calculates effect from stop_time status list (this work is also done in kraken and has to be deleted there) if trip_update.effect == TripEffect.MODIFIED_SERVICE.name: trip_update.effect = get_effect_from_modification_type( highest_st_status) return trip_update
def _make_trip_updates(self, input_trip_update, data_time): """ If trip_update.stop_time_updates is not a strict ending subset of vj.stop_times we reject the trip update On the other hand: 1. For the stop point present in trip_update.stop_time_updates we create a trip_update merging informations with that of navitia stop 2. For the first stop point absent in trip_update.stop_time_updates we create a stop_time_update with no delay for that stop """ vjs = self._get_navitia_vjs(input_trip_update.trip, data_time=data_time) trip_updates = [] for vj in vjs: trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor is_tu_valid = True vj_stop_order = len(vj.navitia_vj.get('stop_times', [])) - 1 for vj_stop, tu_stop in itertools.izip_longest( reversed(vj.navitia_vj.get('stop_times', [])), reversed(input_trip_update.stop_time_update)): if vj_stop is None: is_tu_valid = False break vj_stop_point = vj_stop.get('stop_point') if vj_stop_point is None: is_tu_valid = False break if tu_stop is not None: if self._get_stop_code(vj_stop_point) != tu_stop.stop_id: is_tu_valid = False break tu_stop.stop_sequence = vj_stop_order st_update = self._make_stoptime_update( tu_stop, vj_stop_point) if st_update is not None: trip_update.stop_time_updates.append(st_update) else: #Initialize stops absent in trip_updates but present in vj st_update = self._init_stop_update(vj_stop_point, vj_stop_order) if st_update is not None: trip_update.stop_time_updates.append(st_update) vj_stop_order -= 1 if is_tu_valid: #Since vj.stop_times are managed in reversed order, we re sort stop_time_updates by order. trip_update.stop_time_updates.sort( cmp=lambda x, y: cmp(x.order, y.order)) trip_updates.append(trip_update) else: self.log.error( 'stop_time_update do not match with stops in navitia for trip : {} timestamp: {}' .format(input_trip_update.trip.trip_id, calendar.timegm(data_time.utctimetuple()))) record_internal_failure( 'stop_time_update do not match with stops in navitia', contributor=self.contributor) del trip_update.stop_time_updates[:] return trip_updates
def _make_trip_update(self, vj, json_train): """ create the new TripUpdate object Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md """ logger = logging.getLogger(__name__) trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor trip_message_id = get_value(json_train, 'idMotifInterneReference', nullable=True) if trip_message_id: trip_update.message = self.message_handler.get_message(index=trip_message_id) trip_status = get_value(json_train, 'statutOperationnel') if trip_status == 'SUPPRIMEE': # the whole trip is deleted trip_update.status = 'delete' trip_update.stop_time_updates = [] return trip_update elif trip_status == 'AJOUTEE': # the trip is created from scratch # not handled yet self._record_and_log(logger, 'nouvelleVersion/statutOperationnel == "AJOUTEE" is not handled (yet)') return trip_update # all other status is considered an 'update' of the trip trip_update.status = 'update' pdps = _retrieve_interesting_pdp(get_value(json_train, 'listePointDeParcours')) # manage realtime information stop_time by stop_time for pdp in pdps: # retrieve navitia's stop_time information corresponding to the current COTS pdp nav_st, log_dict = self._get_navitia_stop_time(pdp, vj.navitia_vj) if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_st is None: continue nav_stop = nav_st.get('stop_point', {}) st_update = model.StopTimeUpdate(nav_stop) trip_update.stop_time_updates.append(st_update) # using the message from departure-time in priority, if absent fallback on arrival-time's message st_message_id = get_value(pdp, 'idMotifInterneDepartReference', nullable=True) if not st_message_id: st_message_id = get_value(pdp, 'idMotifInterneArriveeReference', nullable=True) if st_message_id: st_update.message = self.message_handler.get_message(index=st_message_id) _status_map = {'Arrivee': 'arrival_status', 'Depart': 'departure_status'} _delay_map = {'Arrivee': 'arrival_delay', 'Depart': 'departure_delay'} # compute realtime information and fill st_update for arrival and departure for arrival_departure_toggle in ['Arrivee', 'Depart']: cots_traveler_time = get_value(pdp, 'horaireVoyageur{}'.format(arrival_departure_toggle), nullable=True) if cots_traveler_time is None: continue cots_stop_time_status = get_value(cots_traveler_time, 'statutCirculationOPE', nullable=True) if cots_stop_time_status is None: # if no cots_stop_time_status, it is considered an 'update' of the stop_time # (can be a delay, back to normal, normal, ...) cots_ref_planned = get_value(pdp, 'sourceHoraireProjete{}Reference'.format( arrival_departure_toggle), nullable=True) cots_planned_stop_times = get_value(pdp, 'listeHoraireProjete{}'.format(arrival_departure_toggle), nullable=True) cots_planned_stop_time = _retrieve_projected_time(cots_ref_planned, cots_planned_stop_times) if cots_planned_stop_time is None: continue cots_delay = get_value(cots_planned_stop_time, 'pronosticIV', nullable=True) if cots_delay is None: continue setattr(st_update, _status_map[arrival_departure_toggle], 'update') setattr(st_update, _delay_map[arrival_departure_toggle], as_duration(cots_delay)) elif cots_stop_time_status == 'SUPPRESSION': # partial delete setattr(st_update, _status_map[arrival_departure_toggle], 'delete') elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT': # stop_time is replaced by another one self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == ' '"{}" is not handled completely (yet), only removal' .format(cots_stop_time_status)) setattr(st_update, _status_map[arrival_departure_toggle], 'delete') elif cots_stop_time_status == 'CREATION': # new stop_time added self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == ' '"{}" is not handled (yet)'.format(cots_stop_time_status)) elif cots_stop_time_status == 'DETOURNEMENT': # new stop_time added also? self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == ' '"{}" is not handled (yet)'.format(cots_stop_time_status)) else: raise InvalidArguments('invalid value {} for field horaireVoyageur{}/statutCirculationOPE'. format(cots_stop_time_status, arrival_departure_toggle)) return trip_update
def setup_database(): """ we create two realtime_updates with the same vj but for different date and return a vj for navitia """ with app.app_context(): vj1 = model.VehicleJourney( { 'trip': { 'id': 'vj:1' }, 'stop_times': [{ 'utc_arrival_time': time(9, 0), 'stop_point': { 'stop_area': { 'timezone': 'Europe/Paris' } } }] }, utc.localize(datetime(2015, 11, 4, 8, 0, 0)), utc.localize(datetime(2015, 11, 4, 10, 0, 0))) vj2 = model.VehicleJourney( { 'trip': { 'id': 'vj:2' }, 'stop_times': [{ 'utc_arrival_time': time(9, 0), 'stop_point': { 'stop_area': { 'timezone': 'Europe/Paris' } } }] }, utc.localize(datetime(2015, 11, 4, 8, 0, 0)), utc.localize(datetime(2015, 11, 4, 10, 0, 0))) vj3 = model.VehicleJourney( { 'trip': { 'id': 'vj:3' }, 'stop_times': [{ 'utc_arrival_time': time(9, 0), 'stop_point': { 'stop_area': { 'timezone': 'Europe/Paris' } } }] }, utc.localize(datetime(2015, 11, 4, 8, 0, 0)), utc.localize(datetime(2015, 11, 4, 10, 0, 0))) tu1 = model.TripUpdate(vj1, contributor='realtime.cots') tu2 = model.TripUpdate(vj2, contributor='realtime.cots') tu3 = model.TripUpdate(vj3, contributor='realtime.sherbrooke') rtu1 = model.RealTimeUpdate(None, 'cots', 'realtime.cots') rtu1.created_at = datetime(2015, 11, 4, 6, 32) rtu1.trip_updates.append(tu1) model.db.session.add(rtu1) rtu2 = model.RealTimeUpdate(None, 'cots', contributor='realtime.cots') rtu2.created_at = datetime(2015, 11, 4, 7, 32) rtu2.trip_updates.append(tu2) model.db.session.add(rtu2) rtu3 = model.RealTimeUpdate(None, 'gtfs-rt', contributor='realtime.sherbrooke') rtu3.created_at = datetime(2015, 11, 4, 7, 42) rtu3.trip_updates.append(tu3) model.db.session.add(rtu3) rtu4 = model.RealTimeUpdate( None, connector='gtfs-rt', contributor='realtime.sherbrooke', status='KO', error='No new information destinated to navitia for this gtfs-rt') rtu4.created_at = datetime(2015, 11, 4, 7, 52) model.db.session.add(rtu4) model.db.session.commit()
def _make_trip_update(self, json_train, vj, action_on_trip=ActionOnTrip.NOT_ADDED.name): """ create the new TripUpdate object Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md """ trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor trip_message_id = get_value(json_train, 'idMotifInterneReference', nullable=True) if trip_message_id: trip_update.message = self.message_handler.get_message( index=trip_message_id) cots_company_id = get_value(json_train, 'codeCompagnieTransporteur', nullable=True) or DEFAULT_COMPANY_ID trip_update.company_id = self._get_navitia_company(cots_company_id) trip_status = get_value(json_train, 'statutOperationnel') if trip_status == TripStatus.SUPPRIMEE.name: # the whole trip is deleted trip_update.status = ModificationType.delete.name trip_update.stop_time_updates = [] trip_update.effect = TripEffect.NO_SERVICE.name return trip_update elif action_on_trip != ActionOnTrip.NOT_ADDED.name: # the trip is created from scratch trip_update.effect = TripEffect.ADDITIONAL_SERVICE.name trip_update.status = ModificationType.add.name cots_physical_mode = get_value(json_train, 'indicateurFer', nullable=True) trip_update.physical_mode_id = self._get_navitia_physical_mode( cots_physical_mode) trip_update.headsign = get_value(json_train, 'numeroCourse', nullable=True) # all other status is considered an 'update' of the trip_update and effect is calculated # from stop_time status list. This part is also done in kraken and is to be deleted later # Ordered stop_time status= 'nochange', 'add', 'delete', 'update' # 'nochange' or 'update' -> SIGNIFICANT_DELAYS, add -> MODIFIED_SERVICE, delete = DETOUR else: trip_update.status = ModificationType.update.name trip_update.effect = TripEffect.MODIFIED_SERVICE.name # Initialize stop_time status to nochange highest_st_status = ModificationType.none.name pdps = _retrieve_interesting_pdp( get_value(json_train, 'listePointDeParcours')) # this variable is used to memoize the last stop_time's departure in order to check the stop_time consistency # ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure last_stop_time_depart = None # manage realtime information stop_time by stop_time for pdp in pdps: # retrieve navitia's stop_point corresponding to the current COTS pdp nav_stop, log_dict = self._get_navitia_stop_point( pdp, vj.navitia_vj) projected_stop_time = { 'Arrivee': None, 'Depart': None } # used to check consistency if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_stop is None: continue st_update = model.StopTimeUpdate(nav_stop) trip_update.stop_time_updates.append(st_update) # using the message from departure-time in priority, if absent fallback on arrival-time's message st_message_id = get_value(pdp, 'idMotifInterneDepartReference', nullable=True) if not st_message_id: st_message_id = get_value(pdp, 'idMotifInterneArriveeReference', nullable=True) if st_message_id: st_update.message = self.message_handler.get_message( index=st_message_id) _status_map = { 'Arrivee': 'arrival_status', 'Depart': 'departure_status' } _delay_map = { 'Arrivee': 'arrival_delay', 'Depart': 'departure_delay' } _stop_event_datetime_map = { 'Arrivee': 'arrival', 'Depart': 'departure' } # compute realtime information and fill st_update for arrival and departure for arrival_departure_toggle in ['Arrivee', 'Depart']: cots_traveler_time = get_value( pdp, 'horaireVoyageur{}'.format(arrival_departure_toggle), nullable=True) if cots_traveler_time is None: continue cots_stop_time_status = get_value(cots_traveler_time, 'statutCirculationOPE', nullable=True) if cots_stop_time_status is None: # if no cots_stop_time_status, it is considered an 'update' of the stop_time # (can be a delay, back to normal, normal, ...) cots_base_datetime = _retrieve_stop_event_datetime( cots_traveler_time) if cots_base_datetime: projected_stop_time[ arrival_departure_toggle] = cots_base_datetime cots_delay = _retrieve_stop_event_delay( pdp, arrival_departure_toggle) if cots_delay is not None: # It's an update only if there is delay projected_stop_time[ arrival_departure_toggle] += cots_delay setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.update.name) setattr(st_update, _delay_map[arrival_departure_toggle], cots_delay) # otherwise nothing to do (status none, delay none, time none) elif cots_stop_time_status == 'SUPPRESSION': # partial delete setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.delete.name) elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT': # stop_time is replaced by another one setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.deleted_for_detour.name) elif cots_stop_time_status in ['CREATION', 'DETOURNEMENT']: # new stop_time added cots_base_datetime = _retrieve_stop_event_datetime( cots_traveler_time) if cots_base_datetime: projected_stop_time[ arrival_departure_toggle] = cots_base_datetime cots_delay = _retrieve_stop_event_delay( pdp, arrival_departure_toggle) if cots_delay is not None: projected_stop_time[ arrival_departure_toggle] += cots_delay setattr(st_update, _stop_event_datetime_map[arrival_departure_toggle], projected_stop_time[arrival_departure_toggle]) # delay already added to stop_event datetime setattr(st_update, _delay_map[arrival_departure_toggle], None) if cots_stop_time_status == 'CREATION': # pure add setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.add.name) elif cots_stop_time_status == 'DETOURNEMENT': # add to replace another stop_time setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.added_for_detour.name) else: raise InvalidArguments( 'invalid value {} for field horaireVoyageur{}/statutCirculationOPE' .format(cots_stop_time_status, arrival_departure_toggle)) arr_dep_status = getattr(st_update, _status_map[arrival_departure_toggle], ModificationType.none.name) highest_st_status = get_higher_status(highest_st_status, arr_dep_status) self._check_stop_time_consistency( last_stop_time_depart, projected_stop_time, pdp_code='-'.join(pdp[key] for key in ['cr', 'ci', 'ch'])) last_stop_time_depart = projected_stop_time['Depart'] # Calculates effect from stop_time status list(this work is also done in kraken and has to be deleted) if trip_update.effect == TripEffect.MODIFIED_SERVICE.name: trip_update.effect = get_effect_by_stop_time_status( highest_st_status) return trip_update
def setup_database(): """ we create two realtime_updates with the same vj but for different date and return a vj for navitia """ with app.app_context(): vj1 = model.VehicleJourney( { "trip": { "id": "vj:1" }, "stop_times": [{ "utc_arrival_time": time(9, 0), "stop_point": { "stop_area": { "timezone": "Europe/Paris" } } }], }, datetime(2015, 11, 4, 8, 0, 0), datetime(2015, 11, 4, 10, 0, 0), ) vj2 = model.VehicleJourney( { "trip": { "id": "vj:2" }, "stop_times": [{ "utc_arrival_time": time(9, 0), "stop_point": { "stop_area": { "timezone": "Europe/Paris" } } }], }, datetime(2015, 11, 4, 8, 0, 0), datetime(2015, 11, 4, 10, 0, 0), ) vj3 = model.VehicleJourney( { "trip": { "id": "vj:3" }, "stop_times": [{ "utc_arrival_time": time(9, 0), "stop_point": { "stop_area": { "timezone": "Europe/Paris" } } }], }, datetime(2015, 11, 4, 8, 0, 0), datetime(2015, 11, 4, 10, 0, 0), ) tu1 = model.TripUpdate(vj1, contributor_id=COTS_CONTRIBUTOR_ID) tu2 = model.TripUpdate(vj2, contributor_id=COTS_CONTRIBUTOR_ID) tu3 = model.TripUpdate(vj3, contributor_id=GTFS_CONTRIBUTOR_ID) rtu1 = make_rt_update(None, COTS_CONTRIBUTOR_ID, status="OK") rtu1.created_at = datetime(2015, 11, 4, 6, 32) rtu1.updated_at = datetime(2015, 11, 4, 6, 32) # mock creation, no update done rtu1.trip_updates.append(tu1) model.db.session.add(rtu1) rtu2 = make_rt_update(None, contributor_id=COTS_CONTRIBUTOR_ID, status="OK") rtu2.created_at = datetime(2015, 11, 4, 7, 32) rtu2.updated_at = datetime(2015, 11, 4, 7, 32) rtu2.trip_updates.append(tu2) model.db.session.add(rtu2) rtu3 = make_rt_update(None, contributor_id=GTFS_CONTRIBUTOR_ID, status="OK") rtu3.created_at = datetime(2015, 11, 4, 7, 42) rtu3.updated_at = datetime(2015, 11, 4, 7, 42) rtu3.trip_updates.append(tu3) model.db.session.add(rtu3) rtu4 = save_rt_data_with_error( None, contributor_id=GTFS_CONTRIBUTOR_ID, error="No new information destined for navitia on rt.vroumvroum", is_reprocess_same_data_allowed=False, ) rtu4.created_at = datetime(2015, 11, 4, 7, 52) rtu4.updated_at = datetime(2015, 11, 4, 7, 52) model.db.session.add(rtu4) rtu5 = make_rt_update(None, contributor_id=GTFS_CONTRIBUTOR_DB_ID, status="OK") rtu5.created_at = datetime(2015, 11, 4, 8, 2) rtu5.updated_at = datetime(2015, 11, 4, 8, 2) model.db.session.add(rtu5) rtu6 = make_rt_update(None, contributor_id=COTS_CONTRIBUTOR_DB_ID, status="OK") rtu6.created_at = datetime(2015, 11, 4, 8, 12) rtu6.updated_at = datetime(2015, 11, 4, 8, 12) model.db.session.add(rtu6) rtu_piv = make_rt_update(None, contributor_id=PIV_CONTRIBUTOR_ID, status="OK") rtu_piv.created_at = datetime(2015, 11, 4, 8, 17) rtu_piv.updated_at = datetime(2015, 11, 4, 8, 17) model.db.session.add(rtu_piv) model.db.session.commit()
def _build_trip_update(self, feed): feed_vj = self._get_vj(feed) if feed_vj is None: self.log.info( "Missing object in SIRI-ET-XML-TN feed to evaluate the real-time information; the feed should contain 'EstimatedVehicleJourney' XML object; no 'TripUpdate' will be created" ) return None vj_name = self._get_vj_name(feed_vj) line_ref = self._get_line_ref(feed_vj) origin_dt = self._get_origin_dt(feed_vj) since_dt = origin_dt - timedelta(minutes=60) until_dt = origin_dt + timedelta(minutes=60) if vj_name is None or line_ref is None or origin_dt is None: self.log.info( "Missing information to identify the impacted vehicle journey in SIRI-ET-XML-TN; the feed should contain: 'VehicleJourneyName', 'LineRef' and 'OriginAimedDepartureTime'; no 'TripUpdate' will be created" ) return None vj = self._get_navitia_vj(vj_name, line_ref, since_dt, until_dt) if vj is None: self.log.info( "Failed to find the vehicle journey in navitia from 'VehicleJourneyName={}', 'LineRef={}' and 'OriginAimedDepartureTime={}'; no 'TripUpdate' will be created" .format(vj_name, line_ref, origin_dt)) return None trip_update = model.TripUpdate(vj=vj, contributor_id=self.contributor.id) cancellation = self._get_cancellation(feed_vj) extra_journey = self._get_extra_journey(feed_vj) # Trip cancelled, no more processing is needed if cancellation: trip_update.effect = TripEffect.NO_SERVICE.name trip_update.status = ModificationType.delete.name return trip_update # If neither Cancellation nor ExtraJourney is true, # then we only handle delayed stop times if not cancellation and not extra_journey: trip_update.effect = TripEffect.SIGNIFICANT_DELAYS.name trip_update.status = ModificationType.update.name st_idx = 0 for feed_st in self._iter_stop_times(feed_vj): st_stop_point_ref = self._get_st_stop_point_ref(feed_st) st_departure_dt = self._get_st_departure_dt(feed_st) st_arrival_dt = self._get_st_arrival_dt(feed_st) st_cancellation = self._get_st_cancellation(feed_st) st_extra_call = self._get_st_extra_call(feed_st) st_platform_traversal = self._get_st_platform_traversal( feed_st) if st_platform_traversal: # The stop time is ignored continue if st_cancellation or st_extra_call: # We do not process (yet!) stop_time deletion or addition self.log.info( "Deletion or addition of stops are not yet supported; no 'TripUpdate' will be created" ) return None st = vj.navitia_vj.get("stop_times")[st_idx] sp = st.get("stop_point") base_sp = sp.get("id") if st_stop_point_ref != base_sp: # Stop points from base schedule and feed don't match self.log.info( "Stop from base schedule '{}' don't match the one '{}' in SIRI-ET-XML-TN feed; no 'TripUpdate' will be created" .format(st_stop_point_ref, base_sp)) return None stu = model.StopTimeUpdate(sp) departure_delay = st_departure_dt - datetime.combine( origin_dt.date(), st.get("utc_departure_time")) stu.update_departure(st_departure_dt, departure_delay, ModificationType.update.name) arrival_delay = st_arrival_dt - datetime.combine( origin_dt.date(), st.get("utc_arrival_time")) stu.update_arrival(st_arrival_dt, arrival_delay, ModificationType.update.name) trip_update.stop_time_updates.append(stu) st_idx += 1 return trip_update