def _build_one_stop_time_update(self, gtfsrt_stop, nav_vj, previous_rt_stop_event_time): """ Build a stop time update from given GTFSRT STU, navitia VJ, and check consistency versus previous stop-event datetime (should be ascending) :param gtfsrt_stop: GTFS StopTimeUpdate to convert :param nav_vj: Navitia VJ to be used to retrieve information on stop :param previous_rt_stop_event_time: Previous stop-event datetime :return: a couple (converted STU, last_stop_event_datetime) if conversion was a success and consistent """ # retrieve navitia's stop_point corresponding to the current gtfsrt stop nav_stop, log_dict = self._get_navitia_stop_point(gtfsrt_stop, nav_vj) if log_dict: record_internal_failure(log_dict["log"], contributor=self.contributor.id) log_dict.update({"contributor": self.contributor.id}) self.log.info("metrology", extra=log_dict) if nav_stop is None: # TODO: log and count in NR return None, None # simply skip stop_times at unknown stop points st_update = model.StopTimeUpdate(nav_stop) for event_toggle in ["arrival", "departure"]: previous_rt_stop_event_time = _fill_and_check_one_stop_event( st_update=st_update, event_toggle=event_toggle, gtfsrt_stop=gtfsrt_stop, previous_rt_stop_event_time=previous_rt_stop_event_time, ) return st_update, previous_rt_stop_event_time
def _get_navitia_vj(self, piv_key, train_date, ads, is_trip_addition): self.log.debug("searching for vj {} in navitia".format(piv_key)) # large filter on date mostly to ensure only base-schedule VJ are requested filter = 'vehicle_journey.has_code("rt_piv", "{}")'.format(piv_key) train_datetime = datetime.datetime.combine(train_date, datetime.time(0, 0, 0)) since_dt = train_datetime - datetime.timedelta(days=1) until_dt = train_datetime + datetime.timedelta(days=2) navitia_vjs = self._request_navitia_vehicle_journeys(filter, since_dt, until_dt, depth=2, show_codes=True) if not navitia_vjs: # Last PIV information is always right, so if the VJ doesn't exist, it's an ADD (no matter feed content) navitia_vjs = [_make_navitia_empty_vj(piv_key)] vj = None if len(navitia_vjs) != 1: self.log.info( "Can not match a unique train for key {}".format(piv_key)) record_internal_failure("no unique train", contributor=self.contributor.id) else: navitia_vj = navitia_vjs[0] try: base_vs_rt_error_margin = datetime.timedelta(hours=1) vj_base_start = _get_first_stop_base_datetime( ads, "depart", skip_fully_added_stops=not is_trip_addition) # not even a single stop is a base-schedule stop if not vj_base_start: raise InvalidArguments( "Whole trip is specified as pre-existing, but no stop is specified as pre-existing" ) vj = model.VehicleJourney( navitia_vj, vj_base_start - base_vs_rt_error_margin, vj_base_start + base_vs_rt_error_margin, vj_start_dt=vj_base_start, ) except InvalidArguments as i: raise i except Exception as e: self.log.exception( "Error while creating kirin VJ of {}: {}".format( navitia_vjs[0].get("id"), e)) record_internal_failure("Error while creating kirin VJ", contributor=self.contributor.id) if not vj: raise ObjectNotFound("no train found for key {}".format(piv_key)) return vj
def _get_navitia_vjs(self, headsign_str, since_dt, until_dt): """ Search for navitia's vehicle journeys with given headsigns, in the period provided """ log = logging.getLogger(__name__) vjs = {} # to get the date of the vj we use the start/end of the vj + some tolerance # since the SNCF data and navitia data might not be synchronized extended_since_dt = since_dt - timedelta(hours=1) extended_until_dt = until_dt + timedelta(hours=1) # using a set to deduplicate # one headsign_str (ex: "96320/1") can lead to multiple headsigns (ex: ["96320", "96321"]) # but most of the time (if not always) they refer to the same VJ # (the VJ switches headsign along the way). # So we do one VJ search for each headsign to ensure we get it, then deduplicate VJs for train_number in headsigns(headsign_str): log.debug('searching for vj {} on {} in navitia'.format( train_number, since_dt)) navitia_vjs = self.navitia.vehicle_journeys( q={ 'headsign': train_number, 'since': to_navitia_str(extended_since_dt), 'until': to_navitia_str(extended_until_dt), 'depth': '2', # we need this depth to get the stoptime's stop_area 'show_codes': 'true' # we need the stop_points CRCICH codes }) if not navitia_vjs: logging.getLogger(__name__).info( 'impossible to find train {t} on [{s}, {u}['.format( t=train_number, s=extended_since_dt, u=extended_until_dt)) record_internal_failure('missing train', contributor=self.contributor) for nav_vj in navitia_vjs: vj = model.VehicleJourney(nav_vj, since_dt.date()) vjs[nav_vj['id']] = vj if not vjs: raise ObjectNotFound( 'no train found for headsign(s) {}'.format(headsign_str)) return vjs.values()
def _get_vjs(self, xml_train): log = logging.getLogger(__name__) train_numbers = headsigns(get_value(xml_train, 'NumeroTrain')) # to get the date of the vj we use the start/end of the vj + some tolerance # since the ire data and navitia data might not be synchronized vj_start = as_date( get_value(xml_train, 'OrigineTheoriqueTrain/DateHeureDepart')) since = vj_start - timedelta(hours=1) vj_end = as_date( get_value(xml_train, 'TerminusTheoriqueTrain/DateHeureTerminus')) until = vj_end + timedelta(hours=1) vjs = {} for train_number in train_numbers: log.debug('searching for vj {} on {} in navitia'.format( train_number, vj_start)) navitia_vjs = self.navitia.vehicle_journeys( q={ 'headsign': train_number, 'since': to_str(since), 'until': to_str(until), 'depth': '2', # we need this depth to get the stoptime's stop_area 'show_codes': 'true' # we need the stop_points CRCICH codes }) if not navitia_vjs: logging.getLogger(__name__).info( 'impossible to find train {t} on [{s}, {u}['.format( t=train_number, s=since, u=until)) record_internal_failure('missing train', contributor=self.contributor) for nav_vj in navitia_vjs: vj = model.VehicleJourney(nav_vj, vj_start.date()) vjs[nav_vj['id']] = vj if not vjs: raise ObjectNotFound( 'no train found for headsigns {}'.format(train_numbers)) return vjs.values()
def _get_navitia_vj(self, trip_id, since_dt, until_dt): filter = "vehicle_journey.has_code({}, {})".format(self.stop_code_key, trip_id) navitia_vjs = self._request_navitia_vehicle_journeys(filter, since_dt, until_dt, depth=2) if not navitia_vjs: self.log.info("impossible to find vj {t} on [{s}, {u}]".format(t=trip_id, s=since_dt, u=until_dt)) record_internal_failure("missing vj", contributor=self.contributor.id) return None if len(navitia_vjs) > 1: vj_ids = [vj.get("id") for vj in navitia_vjs] self.log.info( "too many vjs found for {t} on [{s}, {u}]: {ids}".format( t=trip_id, s=since_dt, u=until_dt, ids=vj_ids ) ) record_internal_failure("duplicate vjs", contributor=self.contributor.id) return None nav_vj = navitia_vjs[0] vj = None try: vj = model.VehicleJourney(nav_vj, since_dt, until_dt) except Exception as e: self.log.exception("Error while creating kirin VJ of {}: {}".format(nav_vj.get("id"), e)) record_internal_failure("Error while creating kirin VJ", contributor=self.contributor.id) return vj
def _make_db_vj(self, vj_source_code, since_dt, until_dt): """ Search for navitia's vehicle journeys with given code, in the period provided :param vj_source_code: the code to search for :param since_dt: naive UTC datetime that starts the search period. Typically the supposed datetime of first base-schedule stop_time. :param until_dt: naive UTC datetime that ends the search period. Typically the supposed datetime of last base-schedule stop_time. """ if since_dt.tzinfo is not None or until_dt.tzinfo is not None: raise InternalException( "Invalid datetime provided: must be naive (and UTC)") navitia_vjs = self.navitia.vehicle_journeys( q={ "filter": "vehicle_journey.has_code({}, {})".format( self.stop_code_key, vj_source_code), "since": to_navitia_utc_str(since_dt), "until": to_navitia_utc_str(until_dt), "depth": "2", # we need this depth to get the stoptime's stop_area }) if not navitia_vjs: self.log.info("impossible to find vj {t} on [{s}, {u}]".format( t=vj_source_code, s=since_dt, u=until_dt)) record_internal_failure("missing vj", contributor=self.contributor.id) return [] if len(navitia_vjs) > 1: vj_ids = [vj.get("id") for vj in navitia_vjs] self.log.info( "too many vjs found for {t} on [{s}, {u}]: {ids}".format( t=vj_source_code, s=since_dt, u=until_dt, ids=vj_ids)) record_internal_failure("duplicate vjs", contributor=self.contributor.id) return [] nav_vj = navitia_vjs[0] try: vj = model.VehicleJourney(nav_vj, since_dt, until_dt) return [vj] except Exception as e: self.log.exception( "Error while creating kirin VJ of {}: {}".format( nav_vj.get("id"), e)) record_internal_failure("Error while creating kirin VJ", contributor=self.contributor.id) return []
def _make_db_vj(self, vj_source_code, since_dt, until_dt): """ Search for navitia's vehicle journeys with given code, in the period provided :param vj_source_code: the code to search for :param since_dt: naive UTC datetime that starts the search period. Typically the supposed datetime of first base-schedule stop_time. :param until_dt: naive UTC datetime that ends the search period. Typically the supposed datetime of last base-schedule stop_time. """ filter = "vehicle_journey.has_code({}, {})".format( self.stop_code_key, vj_source_code) navitia_vjs = self._request_navitia_vehicle_journeys(filter, since_dt, until_dt, depth=2) if not navitia_vjs: self.log.info("impossible to find vj {t} on [{s}, {u}]".format( t=vj_source_code, s=since_dt, u=until_dt)) record_internal_failure("missing vj", contributor=self.contributor.id) return [] if len(navitia_vjs) > 1: vj_ids = [vj.get("id") for vj in navitia_vjs] self.log.info( "too many vjs found for {t} on [{s}, {u}]: {ids}".format( t=vj_source_code, s=since_dt, u=until_dt, ids=vj_ids)) record_internal_failure("duplicate vjs", contributor=self.contributor.id) return [] nav_vj = navitia_vjs[0] try: vj = model.VehicleJourney(nav_vj, since_dt, until_dt) return [vj] except Exception as e: self.log.exception( "Error while creating kirin VJ of {}: {}".format( nav_vj.get("id"), e)) record_internal_failure("Error while creating kirin VJ", contributor=self.contributor.id) return []
def _make_db_vj(self, vj_source_code, utc_since_dt, utc_until_dt): navitia_vjs = self.navitia.vehicle_journeys( q={ 'filter': 'vehicle_journey.has_code({}, {})'.format( self.stop_code_key, vj_source_code), 'since': to_str(utc_since_dt), 'until': to_str(utc_until_dt), 'depth': '2', # we need this depth to get the stoptime's stop_area }) if not navitia_vjs: self.log.info('impossible to find vj {t} on [{s}, {u}]'.format( t=vj_source_code, s=utc_since_dt, u=utc_until_dt)) record_internal_failure('missing vj', contributor=self.contributor) return [] if len(navitia_vjs) > 1: vj_ids = [vj.get('id') for vj in navitia_vjs] self.log.info( 'too many vjs found for {t} on [{s}, {u}]: {ids}'.format( t=vj_source_code, s=utc_since_dt, u=utc_until_dt, ids=vj_ids)) record_internal_failure('duplicate vjs', contributor=self.contributor) return [] nav_vj = navitia_vjs[0] try: vj = model.VehicleJourney(nav_vj, utc_since_dt, utc_until_dt) return [vj] except Exception as e: self.log.exception( 'Error while creating kirin VJ of {}: {}'.format( nav_vj.get('id'), e)) record_internal_failure('Error while creating kirin VJ', contributor=self.contributor) return []
def _make_trip_updates(self, input_trip_update, input_data_time): """ If trip_update.stop_time_updates is not a strict ending subset of vj.stop_times we reject the trip update On the other hand: 1. For the stop point present in trip_update.stop_time_updates we create a trip_update merging informations with that of navitia stop 2. For the first stop point absent in trip_update.stop_time_updates we create a stop_time_update with no delay for that stop """ vjs = self._get_navitia_vjs(input_trip_update.trip, input_data_time=input_data_time) trip_updates = [] for vj in vjs: trip_update = model.TripUpdate(vj=vj, contributor_id=self.contributor.id) highest_st_status = ModificationType.none.name is_tu_valid = True vj_stop_order = len(vj.navitia_vj.get("stop_times", [])) - 1 for vj_stop, tu_stop in itertools.zip_longest( reversed(vj.navitia_vj.get("stop_times", [])), reversed(input_trip_update.stop_time_update)): if vj_stop is None: is_tu_valid = False break vj_stop_point = vj_stop.get("stop_point") if vj_stop_point is None: is_tu_valid = False break if tu_stop is not None: if self._get_stop_code(vj_stop_point) != tu_stop.stop_id: is_tu_valid = False break tu_stop.stop_sequence = vj_stop_order st_update = _make_stoptime_update(tu_stop, vj_stop_point) if st_update is not None: trip_update.stop_time_updates.append(st_update) else: # Initialize stops absent in trip_updates but present in vj st_update = _init_stop_update(vj_stop_point, vj_stop_order) if st_update is not None: trip_update.stop_time_updates.append(st_update) for status in [ st_update.departure_status, st_update.arrival_status ]: highest_st_status = get_higher_status( highest_st_status, status) vj_stop_order -= 1 if is_tu_valid: # Since vj.stop_times are managed in reversed order, we re sort stop_time_updates by order. trip_update.stop_time_updates.sort(key=lambda x: x.order) trip_update.effect = get_effect_from_modification_type( highest_st_status) trip_updates.append(trip_update) else: self.log.warning( "stop_time_update do not match with stops in navitia for trip : {} timestamp: {}" .format(input_trip_update.trip.trip_id, calendar.timegm(input_data_time.utctimetuple()))) record_internal_failure( "stop_time_update do not match with stops in navitia", contributor=self.contributor.id) del trip_update.stop_time_updates[:] return trip_updates
def _make_trip_update(self, json_train, vj): trip_update = model.TripUpdate(vj=vj, contributor_id=self.contributor.id) trip_update.headsign = json_train.get("numero") company_id = json_train.get("operateur").get("codeOperateur") trip_update.company_id = self._get_navitia_company_id(company_id) physical_mode = json_train.get("modeTransport").get("typeMode") trip_update.physical_mode_id = self._get_navitia_physical_mode_id( physical_mode) trip_status_type = trip_piv_status_to_effect.get( json_train.get("evenement").get("type"), TripEffect.UNDEFINED) trip_update.message = json_train.get("evenement").get("texte") trip_update.effect = trip_status_type.name if trip_status_type == TripEffect.NO_SERVICE: # the whole trip is deleted trip_update.status = ModificationType.delete.name trip_update.stop_time_updates = [] return trip_update elif trip_status_type == TripEffect.ADDITIONAL_SERVICE: trip_update.status = ModificationType.add.name else: trip_update.status = ModificationType.update.name highest_st_status = ModificationType.none.name ads = get_value(get_value(json_train, "listeArretsDesserte"), "arret") # this variable is used to store the last stop_time's departure in order to check the stop_time consistency # ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure previous_rt_stop_event_time = datetime.datetime.utcfromtimestamp(0) for arret in ads: # retrieve navitia's stop_point corresponding to the current PIV ad nav_stop, log_dict = self._get_navitia_stop_point( arret, vj.navitia_vj) if log_dict: record_internal_failure(log_dict["log"], contributor=self.contributor.id) log_dict.update({"contributor": self.contributor.id}) logging.getLogger(__name__).info("metrology", extra=log_dict) if nav_stop is None: continue # simply skip stop_times at unknown stop areas st_update = model.StopTimeUpdate(nav_stop) trip_update.stop_time_updates.append(st_update) st_update.message = _get_message(arret) for event_toggle in ["arrivee", "depart"]: event = get_value(arret, event_toggle, nullable=True) if event is None: continue piv_disruption = event.get("evenement", {}) piv_event_status = event.get("statutModification", piv_disruption.get("type", None)) if not piv_event_status or piv_event_status in MANAGED_STOP_EVENTS: piv_event_datetime = get_value(event, "dateHeureReelle", nullable=True) event_datetime = str_to_utc_naive_dt( piv_event_datetime) if piv_event_datetime else None if event_datetime: setattr(st_update, STOP_EVENT_DATETIME_MAP[event_toggle], event_datetime) if piv_event_status in ["SUPPRESSION_PARTIELLE"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.delete.name) elif piv_event_status in ["SUPPRESSION_DETOURNEMENT"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.deleted_for_detour.name) elif piv_event_status in ["CREATION"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.add.name) elif piv_event_status in ["CREATION_DETOURNEMENT"]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.added_for_detour.name) elif trip_status_type == TripEffect.ADDITIONAL_SERVICE: # In this case we want to create schedules taking into account the delay # without adding the delay a second time setattr(st_update, STATUS_MAP[event_toggle], ModificationType.add.name) elif piv_event_status in [ "RETARD_OBSERVE", "RETARD_PROJETE" ]: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.update.name) if piv_disruption: piv_event_delay = piv_disruption.get( "retard", {}).get("duree", 0) setattr(st_update, DELAY_MAP[event_toggle], as_duration(piv_event_delay * 60)) # minutes else: setattr(st_update, STATUS_MAP[event_toggle], ModificationType.none.name) # otherwise let those be none else: raise InvalidArguments( "invalid value {s} for field {t}/statutModification or {t}/evenement/type" .format(s=piv_event_status, t=event_toggle)) event_status = getattr(st_update, STATUS_MAP[event_toggle], ModificationType.none.name) highest_st_status = get_higher_status(highest_st_status, event_status) mdi = (arret[STOP_EVENT_MDI_MAP[event_toggle]] if arret.get( STOP_EVENT_MDI_MAP[event_toggle]) else False) if _is_stop_event_served(event_datetime, event_status, mdi): if previous_rt_stop_event_time > event_datetime: raise InvalidArguments( "invalid feed: stop_point's({}) time is not consistent" .format( get_value(get_value(arret, "emplacement"), "code"))) previous_rt_stop_event_time = event_datetime # Calculates effect from stop_time status list (this work is also done in kraken and has to be deleted there) if trip_update.effect == TripEffect.MODIFIED_SERVICE.name: trip_update.effect = get_effect_from_modification_type( highest_st_status) return trip_update
def _get_navitia_vjs(self, headsign_str, since_dt, until_dt, action_on_trip=ActionOnTrip.NOT_ADDED.name): """ Search for navitia's vehicle journeys with given headsigns, in the period provided :param headsign_str: the headsigns to search for (can be multiple expressed in one string, like "2512/3") :param since_dt: naive UTC datetime that starts the search period. Typically the supposed datetime of first base-schedule stop_time. :param until_dt: naive UTC datetime that ends the search period. Typically the supposed datetime of last base-schedule stop_time. :param action_on_trip: action to be performed on trip. This param is used to do consistency check """ if (since_dt is None) or (until_dt is None): return [] if since_dt.tzinfo is not None or until_dt.tzinfo is not None: raise InternalException( "Invalid datetime provided: must be naive (and UTC)") vjs = {} # to get the date of the vj we use the start/end of the vj + some tolerance # since the SNCF data and navitia data might not be synchronized extended_since_dt = since_dt - SNCF_SEARCH_MARGIN extended_until_dt = until_dt + SNCF_SEARCH_MARGIN # using a set to deduplicate # one headsign_str (ex: "96320/1") can lead to multiple headsigns (ex: ["96320", "96321"]) # but most of the time (if not always) they refer to the same VJ # (the VJ switches headsign along the way). # So we do one VJ search for each headsign to ensure we get it, then deduplicate VJs for train_number in headsigns(headsign_str): self.log.debug( "searching for vj {} during period [{} - {}] in navitia". format(train_number, extended_since_dt, extended_until_dt)) navitia_vjs = self.navitia.vehicle_journeys( q={ "headsign": train_number, "since": to_navitia_utc_str(extended_since_dt), "until": to_navitia_utc_str(extended_until_dt), "data_freshness": "base_schedule", "depth": "2", # we need this depth to get the stoptime's stop_area "show_codes": "true", # we need the stop_points CRCICH codes }) # Consistency check on action applied to trip if action_on_trip == ActionOnTrip.NOT_ADDED.name: if not navitia_vjs: self.log.info( "impossible to find train {t} on [{s}, {u}[".format( t=train_number, s=extended_since_dt, u=extended_until_dt)) record_internal_failure("missing train", contributor=self.contributor.id) else: if action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name and navitia_vjs: raise InvalidArguments( "Invalid action, trip {} already present in navitia". format(train_number)) navitia_vjs = [make_navitia_empty_vj(train_number)] for nav_vj in navitia_vjs: try: vj = model.VehicleJourney(nav_vj, extended_since_dt, extended_until_dt, vj_start_dt=since_dt) vjs[nav_vj["id"]] = vj except Exception as e: self.log.exception( "Error while creating kirin VJ of {}: {}".format( nav_vj.get("id"), e)) record_internal_failure("Error while creating kirin VJ", contributor=self.contributor.id) if not vjs: raise ObjectNotFound( "no train found for headsign(s) {}".format(headsign_str)) return vjs.values()
def merge_trip_updates(self, navitia_vj, db_trip_update, new_trip_update): """ Steps: 1. Build TripUpdate info resulting of merge 2. Adjust TripUpdate info to have it be self-consistent (call adjust_trip_update_consistency()) 3. If resulting TripUpdate info are new compared to the one from previous RT info: send it NB: * Working with ORM objects directly: no persistence of new object before knowing it's final version and it's not a duplicate (performance and db unicity constraints) """ # * Retrieve information available in the 3 StopTimeUpdate lists * circulation_date = new_trip_update.vj.get_circulation_date() new_stus = new_trip_update.stop_time_updates # base-schedule STU list nav_stus = convert_nav_stop_list_to_stu_list(navitia_vj.get("stop_times", []), circulation_date) # last info known about STU in trip (before processing new feed): # either from previous RT feed or base-schedule old_stus = db_trip_update.stop_time_updates if db_trip_update else nav_stus # * Build matches between stus in different lists * old_to_nav = map_old_stu_to_nav_stu(old_stus=old_stus, nav_stus=nav_stus) if old_to_nav is None: # If nav_stus is not strictly contained into old_stus, then base-schedule was probably reloaded. # So let's restart from scratch using base-schedule fail_str = "Warning: previous RT result doesn't contain base-schedule" log_str = "{f}: navitia_vj.id={id}".format(f=fail_str, id=navitia_vj.get("id")) self.log.warning(log_str) record_internal_failure(fail_str, contributor=self.contributor.id, reason=log_str) old_stus = nav_stus old_to_nav = map_old_stu_to_nav_stu(old_stus=old_stus, nav_stus=nav_stus) new_to_old = list_match_new_stu_to_old_stu(new_stus=new_stus, old_stus=old_stus) # * Build resulting STU list * # keeps track of what was processed in previous STU info old_stus_unprocessed_start = new_to_old[0][1] if new_to_old else len(old_stus) # Old STUs before the first STU mentioned in new_stus: # Regular behavior is to keep them as-is (considered "archived") res_stus = old_stus[:old_stus_unprocessed_start] # Only exception is if trip is entirely deleted: keep only stops in nav_stus if new_trip_update.effect == TripEffect.NO_SERVICE.name: res_stus = [ old_stus[idx] for idx in filter(lambda i: i in old_to_nav, range(0, old_stus_unprocessed_start)) ] new_stus_unprocessed_start = 0 propagated_delay = as_duration(0) # populate with new_stus (iterating on matches and filling voids in matches-intervals) for new_index, old_index in new_to_old: stus_until_match, last_delay = build_stops_until_match( match_new_index=new_index, match_old_index=old_index, new_stus_unprocessed_start=new_stus_unprocessed_start, old_stus_unprocessed_start=old_stus_unprocessed_start, new_stus=new_stus, nav_stus=nav_stus, old_to_nav=old_to_nav, result_idx_offset=len(res_stus), propagated_delay=propagated_delay, ) res_stus.extend(stus_until_match) propagated_delay = last_delay # Remember progress new_stus_unprocessed_start = new_index + 1 old_stus_unprocessed_start = old_index + 1 # Finalize populating unmatched STUs in old_stus and reversely those unmatched in new_stus stus_until_end, _last_delay = create_intermediate_stus_before_match( match_new_index=len(new_stus), match_old_index=len(old_stus), new_stus_unprocessed_start=new_stus_unprocessed_start, old_stus_unprocessed_start=old_stus_unprocessed_start, new_stus=new_stus, nav_stus=nav_stus, old_to_nav=old_to_nav, result_idx_offset=len(res_stus), propagated_delay=propagated_delay, ) res_stus.extend(stus_until_end) # adjust consistency for resulting trip_update adjust_gtfsrt_trip_update_consistency(new_trip_update, res_stus, nav_stus) return build_trip_update_if_modified( db_trip_update, old_stus, new_trip_update, res_stus, self.contributor.id )
def _make_db_vj(self, vj_source_code, since, until): navitia_vjs = self.navitia.vehicle_journeys( q={ 'filter': 'vehicle_journey.has_code({}, {})'.format( self.stop_code_key, vj_source_code), 'since': to_str(since), 'until': to_str(until), 'depth': '2', # we need this depth to get the stoptime's stop_area }) if not navitia_vjs: self.log.info('impossible to find vj {t} on [{s}, {u}]'.format( t=vj_source_code, s=since, u=until)) record_internal_failure('missing vj', contributor=self.contributor) return [] if len(navitia_vjs) > 1: vj_ids = [vj.get('id') for vj in navitia_vjs] self.log.info( 'too many vjs found for {t} on [{s}, {u}]: {ids}'.format( t=vj_source_code, s=since, u=until, ids=vj_ids)) record_internal_failure('duplicate vjs', contributor=self.contributor) return [] nav_vj = navitia_vjs[0] # Now we compute the real circulate_date of VJ from since, until and vj's first stop_time # We do this to prevent cases like pass midnight when [since, until] is too large # we need local timezone circulate_date (and it's sometimes different from UTC date) first_stop_time = nav_vj.get('stop_times', [{}])[0] tzinfo = get_timezone(first_stop_time) # 'since' and 'until' must have a timezone before being converted to local timezone local_since = pytz.utc.localize(since).astimezone(tzinfo) local_until = pytz.utc.localize(until).astimezone(tzinfo) circulate_date = None if local_since.date() == local_until.date(): circulate_date = local_since.date() else: arrival_time = first_stop_time['arrival_time'] # At first, we suppose that the circulate_date is local_since's date if local_since <= tzinfo.localize( datetime.datetime.combine(local_since.date(), arrival_time)) <= local_until: circulate_date = local_since.date() elif local_since <= tzinfo.localize( datetime.datetime.combine(local_until.date(), arrival_time)) <= local_until: circulate_date = local_until.date() if circulate_date is None: self.log.error( 'impossible to calculate the circulate date (local) of vj: {}'. format(nav_vj.get('id'))) record_internal_failure( 'impossible to calculate the circulate date of vj', contributor=self.contributor) return [] try: vj = model.VehicleJourney(nav_vj, circulate_date) return [vj] except Exception as e: self.log.exception( 'Error while creating kirin VJ of {}: {}'.format( nav_vj.get('id'), e)) record_internal_failure('Error while creating kirin VJ', contributor=self.contributor) return []
def _make_trip_updates(self, input_trip_update, data_time): """ If trip_update.stop_time_updates is not a strict ending subset of vj.stop_times we reject the trip update On the other hand: 1. For the stop point present in trip_update.stop_time_updates we create a trip_update merging informations with that of navitia stop 2. For the first stop point absent in trip_update.stop_time_updates we create a stop_time_update with no delay for that stop """ vjs = self._get_navitia_vjs(input_trip_update.trip, data_time=data_time) trip_updates = [] for vj in vjs: trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor is_tu_valid = True vj_stop_order = len(vj.navitia_vj.get('stop_times', [])) - 1 for vj_stop, tu_stop in itertools.izip_longest( reversed(vj.navitia_vj.get('stop_times', [])), reversed(input_trip_update.stop_time_update)): if vj_stop is None: is_tu_valid = False break vj_stop_point = vj_stop.get('stop_point') if vj_stop_point is None: is_tu_valid = False break if tu_stop is not None: if self._get_stop_code(vj_stop_point) != tu_stop.stop_id: is_tu_valid = False break tu_stop.stop_sequence = vj_stop_order st_update = self._make_stoptime_update( tu_stop, vj_stop_point) if st_update is not None: trip_update.stop_time_updates.append(st_update) else: #Initialize stops absent in trip_updates but present in vj st_update = self._init_stop_update(vj_stop_point, vj_stop_order) if st_update is not None: trip_update.stop_time_updates.append(st_update) vj_stop_order -= 1 if is_tu_valid: #Since vj.stop_times are managed in reversed order, we re sort stop_time_updates by order. trip_update.stop_time_updates.sort( cmp=lambda x, y: cmp(x.order, y.order)) trip_updates.append(trip_update) else: self.log.error( 'stop_time_update do not match with stops in navitia for trip : {} timestamp: {}' .format(input_trip_update.trip.trip_id, calendar.timegm(data_time.utctimetuple()))) record_internal_failure( 'stop_time_update do not match with stops in navitia', contributor=self.contributor) del trip_update.stop_time_updates[:] return trip_updates
def _make_trip_update(self, vj, json_train): """ create the new TripUpdate object Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md """ logger = logging.getLogger(__name__) trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor trip_message_id = get_value(json_train, 'idMotifInterneReference', nullable=True) if trip_message_id: trip_update.message = self.message_handler.get_message(index=trip_message_id) trip_status = get_value(json_train, 'statutOperationnel') if trip_status == 'SUPPRIMEE': # the whole trip is deleted trip_update.status = 'delete' trip_update.stop_time_updates = [] return trip_update elif trip_status == 'AJOUTEE': # the trip is created from scratch # not handled yet self._record_and_log(logger, 'nouvelleVersion/statutOperationnel == "AJOUTEE" is not handled (yet)') return trip_update # all other status is considered an 'update' of the trip trip_update.status = 'update' pdps = _retrieve_interesting_pdp(get_value(json_train, 'listePointDeParcours')) # manage realtime information stop_time by stop_time for pdp in pdps: # retrieve navitia's stop_time information corresponding to the current COTS pdp nav_st, log_dict = self._get_navitia_stop_time(pdp, vj.navitia_vj) if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_st is None: continue nav_stop = nav_st.get('stop_point', {}) st_update = model.StopTimeUpdate(nav_stop) trip_update.stop_time_updates.append(st_update) # using the message from departure-time in priority, if absent fallback on arrival-time's message st_message_id = get_value(pdp, 'idMotifInterneDepartReference', nullable=True) if not st_message_id: st_message_id = get_value(pdp, 'idMotifInterneArriveeReference', nullable=True) if st_message_id: st_update.message = self.message_handler.get_message(index=st_message_id) _status_map = {'Arrivee': 'arrival_status', 'Depart': 'departure_status'} _delay_map = {'Arrivee': 'arrival_delay', 'Depart': 'departure_delay'} # compute realtime information and fill st_update for arrival and departure for arrival_departure_toggle in ['Arrivee', 'Depart']: cots_traveler_time = get_value(pdp, 'horaireVoyageur{}'.format(arrival_departure_toggle), nullable=True) if cots_traveler_time is None: continue cots_stop_time_status = get_value(cots_traveler_time, 'statutCirculationOPE', nullable=True) if cots_stop_time_status is None: # if no cots_stop_time_status, it is considered an 'update' of the stop_time # (can be a delay, back to normal, normal, ...) cots_ref_planned = get_value(pdp, 'sourceHoraireProjete{}Reference'.format( arrival_departure_toggle), nullable=True) cots_planned_stop_times = get_value(pdp, 'listeHoraireProjete{}'.format(arrival_departure_toggle), nullable=True) cots_planned_stop_time = _retrieve_projected_time(cots_ref_planned, cots_planned_stop_times) if cots_planned_stop_time is None: continue cots_delay = get_value(cots_planned_stop_time, 'pronosticIV', nullable=True) if cots_delay is None: continue setattr(st_update, _status_map[arrival_departure_toggle], 'update') setattr(st_update, _delay_map[arrival_departure_toggle], as_duration(cots_delay)) elif cots_stop_time_status == 'SUPPRESSION': # partial delete setattr(st_update, _status_map[arrival_departure_toggle], 'delete') elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT': # stop_time is replaced by another one self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == ' '"{}" is not handled completely (yet), only removal' .format(cots_stop_time_status)) setattr(st_update, _status_map[arrival_departure_toggle], 'delete') elif cots_stop_time_status == 'CREATION': # new stop_time added self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == ' '"{}" is not handled (yet)'.format(cots_stop_time_status)) elif cots_stop_time_status == 'DETOURNEMENT': # new stop_time added also? self._record_and_log(logger, 'nouvelleVersion/listePointDeParcours/statutCirculationOPE == ' '"{}" is not handled (yet)'.format(cots_stop_time_status)) else: raise InvalidArguments('invalid value {} for field horaireVoyageur{}/statutCirculationOPE'. format(cots_stop_time_status, arrival_departure_toggle)) return trip_update
def _record_and_log(self, logger, log_str): log_dict = {'log': log_str} record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logger.info('metrology', extra=log_dict)
def _make_trip_update(self, json_train, vj, action_on_trip=ActionOnTrip.NOT_ADDED.name): """ create the new TripUpdate object Following the COTS spec: https://github.com/CanalTP/kirin/blob/master/documentation/cots_connector.md """ trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor trip_message_id = get_value(json_train, 'idMotifInterneReference', nullable=True) if trip_message_id: trip_update.message = self.message_handler.get_message( index=trip_message_id) cots_company_id = get_value(json_train, 'codeCompagnieTransporteur', nullable=True) or DEFAULT_COMPANY_ID trip_update.company_id = self._get_navitia_company(cots_company_id) trip_status = get_value(json_train, 'statutOperationnel') if trip_status == TripStatus.SUPPRIMEE.name: # the whole trip is deleted trip_update.status = ModificationType.delete.name trip_update.stop_time_updates = [] trip_update.effect = TripEffect.NO_SERVICE.name return trip_update elif action_on_trip != ActionOnTrip.NOT_ADDED.name: # the trip is created from scratch trip_update.effect = TripEffect.ADDITIONAL_SERVICE.name trip_update.status = ModificationType.add.name cots_physical_mode = get_value(json_train, 'indicateurFer', nullable=True) trip_update.physical_mode_id = self._get_navitia_physical_mode( cots_physical_mode) trip_update.headsign = get_value(json_train, 'numeroCourse', nullable=True) # all other status is considered an 'update' of the trip_update and effect is calculated # from stop_time status list. This part is also done in kraken and is to be deleted later # Ordered stop_time status= 'nochange', 'add', 'delete', 'update' # 'nochange' or 'update' -> SIGNIFICANT_DELAYS, add -> MODIFIED_SERVICE, delete = DETOUR else: trip_update.status = ModificationType.update.name trip_update.effect = TripEffect.MODIFIED_SERVICE.name # Initialize stop_time status to nochange highest_st_status = ModificationType.none.name pdps = _retrieve_interesting_pdp( get_value(json_train, 'listePointDeParcours')) # this variable is used to memoize the last stop_time's departure in order to check the stop_time consistency # ex. stop_time[i].arrival/departure must be greater than stop_time[i-1].departure last_stop_time_depart = None # manage realtime information stop_time by stop_time for pdp in pdps: # retrieve navitia's stop_point corresponding to the current COTS pdp nav_stop, log_dict = self._get_navitia_stop_point( pdp, vj.navitia_vj) projected_stop_time = { 'Arrivee': None, 'Depart': None } # used to check consistency if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_stop is None: continue st_update = model.StopTimeUpdate(nav_stop) trip_update.stop_time_updates.append(st_update) # using the message from departure-time in priority, if absent fallback on arrival-time's message st_message_id = get_value(pdp, 'idMotifInterneDepartReference', nullable=True) if not st_message_id: st_message_id = get_value(pdp, 'idMotifInterneArriveeReference', nullable=True) if st_message_id: st_update.message = self.message_handler.get_message( index=st_message_id) _status_map = { 'Arrivee': 'arrival_status', 'Depart': 'departure_status' } _delay_map = { 'Arrivee': 'arrival_delay', 'Depart': 'departure_delay' } _stop_event_datetime_map = { 'Arrivee': 'arrival', 'Depart': 'departure' } # compute realtime information and fill st_update for arrival and departure for arrival_departure_toggle in ['Arrivee', 'Depart']: cots_traveler_time = get_value( pdp, 'horaireVoyageur{}'.format(arrival_departure_toggle), nullable=True) if cots_traveler_time is None: continue cots_stop_time_status = get_value(cots_traveler_time, 'statutCirculationOPE', nullable=True) if cots_stop_time_status is None: # if no cots_stop_time_status, it is considered an 'update' of the stop_time # (can be a delay, back to normal, normal, ...) cots_base_datetime = _retrieve_stop_event_datetime( cots_traveler_time) if cots_base_datetime: projected_stop_time[ arrival_departure_toggle] = cots_base_datetime cots_delay = _retrieve_stop_event_delay( pdp, arrival_departure_toggle) if cots_delay is not None: # It's an update only if there is delay projected_stop_time[ arrival_departure_toggle] += cots_delay setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.update.name) setattr(st_update, _delay_map[arrival_departure_toggle], cots_delay) # otherwise nothing to do (status none, delay none, time none) elif cots_stop_time_status == 'SUPPRESSION': # partial delete setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.delete.name) elif cots_stop_time_status == 'SUPPRESSION_DETOURNEMENT': # stop_time is replaced by another one setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.deleted_for_detour.name) elif cots_stop_time_status in ['CREATION', 'DETOURNEMENT']: # new stop_time added cots_base_datetime = _retrieve_stop_event_datetime( cots_traveler_time) if cots_base_datetime: projected_stop_time[ arrival_departure_toggle] = cots_base_datetime cots_delay = _retrieve_stop_event_delay( pdp, arrival_departure_toggle) if cots_delay is not None: projected_stop_time[ arrival_departure_toggle] += cots_delay setattr(st_update, _stop_event_datetime_map[arrival_departure_toggle], projected_stop_time[arrival_departure_toggle]) # delay already added to stop_event datetime setattr(st_update, _delay_map[arrival_departure_toggle], None) if cots_stop_time_status == 'CREATION': # pure add setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.add.name) elif cots_stop_time_status == 'DETOURNEMENT': # add to replace another stop_time setattr(st_update, _status_map[arrival_departure_toggle], ModificationType.added_for_detour.name) else: raise InvalidArguments( 'invalid value {} for field horaireVoyageur{}/statutCirculationOPE' .format(cots_stop_time_status, arrival_departure_toggle)) arr_dep_status = getattr(st_update, _status_map[arrival_departure_toggle], ModificationType.none.name) highest_st_status = get_higher_status(highest_st_status, arr_dep_status) self._check_stop_time_consistency( last_stop_time_depart, projected_stop_time, pdp_code='-'.join(pdp[key] for key in ['cr', 'ci', 'ch'])) last_stop_time_depart = projected_stop_time['Depart'] # Calculates effect from stop_time status list(this work is also done in kraken and has to be deleted) if trip_update.effect == TripEffect.MODIFIED_SERVICE.name: trip_update.effect = get_effect_by_stop_time_status( highest_st_status) return trip_update
def _make_trip_update(self, vj, xml_modification): """ create the TripUpdate object """ trip_update = model.TripUpdate(vj=vj) trip_update.contributor = self.contributor delay = xml_modification.find('HoraireProjete') if delay: trip_update.status = 'update' for downstream_point in delay.iter('PointAval'): # we need only to consider the station if not as_bool(get_value(downstream_point, 'IndicateurPRGare')): continue nav_st, log_dict = self._get_navitia_stop_time( downstream_point, vj.navitia_vj) if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_st is None: continue nav_stop = nav_st.get('stop_point', {}) dep_delay, dep_status = self._get_delay( downstream_point.find('TypeHoraire/Depart')) arr_delay, arr_status = self._get_delay( downstream_point.find('TypeHoraire/Arrivee')) message = get_value(downstream_point, 'MotifExterne', nullabe=True) st_update = model.StopTimeUpdate(nav_stop, departure_delay=dep_delay, arrival_delay=arr_delay, dep_status=dep_status, arr_status=arr_status, message=message) trip_update.stop_time_updates.append(st_update) removal = xml_modification.find('Suppression') if removal: xml_prdebut = removal.find('PRDebut') if get_value(removal, 'TypeSuppression') == 'T': trip_update.status = 'delete' trip_update.stop_time_updates = [] elif get_value(removal, 'TypeSuppression') == 'P': # it's a partial delete trip_update.status = 'update' deleted_points = itertools.chain([removal.find('PRDebut')], removal.iter('PointSupprime'), [removal.find('PRFin')]) for deleted_point in deleted_points: # we need only to consider the stations if not as_bool(get_value(deleted_point, 'IndicateurPRGare')): continue nav_st, log_dict = self._get_navitia_stop_time( deleted_point, vj.navitia_vj) if log_dict: record_internal_failure(log_dict['log'], contributor=self.contributor) log_dict.update({'contributor': self.contributor}) logging.getLogger(__name__).info('metrology', extra=log_dict) if nav_st is None: continue nav_stop = nav_st.get('stop_point', {}) # if the <Depart>/<Arrivee> tags are there, the departure/arrival has been deleted # regardless of the <Etat> tag dep_deleted = deleted_point.find( 'TypeHoraire/Depart') is not None arr_deleted = deleted_point.find( 'TypeHoraire/Arrivee') is not None dep_status = 'delete' if dep_deleted else 'none' arr_status = 'delete' if arr_deleted else 'none' message = get_value(deleted_point, 'MotifExterne', nullabe=True) st_update = model.StopTimeUpdate(nav_stop, dep_status=dep_status, arr_status=arr_status, message=message) trip_update.stop_time_updates.append(st_update) if xml_prdebut: trip_update.message = get_value(xml_prdebut, 'MotifExterne', nullabe=True) return trip_update
def _get_navitia_vjs(self, headsign_str, utc_since_dt, utc_until_dt, action_on_trip=ActionOnTrip.NOT_ADDED.name): """ Search for navitia's vehicle journeys with given headsigns, in the period provided :param utc_since_dt: UTC datetime that starts the search period. Typically the supposed datetime of first base-schedule stop_time. :param utc_until_dt: UTC datetime that ends the search period. Typically the supposed datetime of last base-schedule stop_time. """ log = logging.getLogger(__name__) if (utc_since_dt is None) or (utc_until_dt is None): return [] vjs = {} # to get the date of the vj we use the start/end of the vj + some tolerance # since the SNCF data and navitia data might not be synchronized extended_since_dt = utc_since_dt - SNCF_SEARCH_MARGIN extended_until_dt = utc_until_dt + SNCF_SEARCH_MARGIN # using a set to deduplicate # one headsign_str (ex: "96320/1") can lead to multiple headsigns (ex: ["96320", "96321"]) # but most of the time (if not always) they refer to the same VJ # (the VJ switches headsign along the way). # So we do one VJ search for each headsign to ensure we get it, then deduplicate VJs for train_number in headsigns(headsign_str): log.debug('searching for vj {} during period [{} - {}] in navitia'. format(train_number, extended_since_dt, extended_until_dt)) navitia_vjs = self.navitia.vehicle_journeys( q={ 'headsign': train_number, 'since': to_navitia_str(extended_since_dt), 'until': to_navitia_str(extended_until_dt), 'depth': '2', # we need this depth to get the stoptime's stop_area 'show_codes': 'true' # we need the stop_points CRCICH codes }) if action_on_trip == ActionOnTrip.NOT_ADDED.name: if not navitia_vjs: logging.getLogger(__name__).info( 'impossible to find train {t} on [{s}, {u}['.format( t=train_number, s=extended_since_dt, u=extended_until_dt)) record_internal_failure('missing train', contributor=self.contributor) else: if action_on_trip == ActionOnTrip.FIRST_TIME_ADDED.name and navitia_vjs: raise InvalidArguments( 'Invalid action, trip {} already present in navitia'. format(train_number)) navitia_vjs = [make_navitia_empty_vj(train_number)] for nav_vj in navitia_vjs: try: vj = model.VehicleJourney(nav_vj, extended_since_dt, extended_until_dt, vj_start_dt=utc_since_dt) vjs[nav_vj['id']] = vj except Exception as e: logging.getLogger(__name__).exception( 'Error while creating kirin VJ of {}: {}'.format( nav_vj.get('id'), e)) record_internal_failure('Error while creating kirin VJ', contributor=self.contributor) if not vjs: raise ObjectNotFound( 'no train found for headsign(s) {}'.format(headsign_str)) return vjs.values()