def get_relevant_service_ids(tracker_id):
result = load_by_key(get_train_tracker_relevant_services_key(tracker_id))
if not result:
start_date = load_by_key(get_train_tracker_day(tracker_id))
relevant_services = gtfs.models.Service.objects.filter(start_date \
= start_date)
result = [x[0] for x in relevant_services.all().values_list(\
'service_id')]
save_by_key(get_train_tracker_relevant_services_key(tracker_id),\
result)
return result
def add_report(tracker_id, report):
logger.info('adding report={}'.format(report))
is_updated_stop_time = False
report_id = cl.incr(get_train_tracker_report_id_key(tracker_id))
updated_report_id_key = get_train_tracker_updated_report_id_key(
tracker_id)
# We try to update a report only if no report was updated with a more recent
# report_id than us. If one was updated, we don't update at all.
# This is done using check-and-set (see http://redis.io/topics/transactions)
done = False
while not done:
logger.info('watching {}'.format(updated_report_id_key))
p.watch(updated_report_id_key)
updated_report_id = load_by_key(updated_report_id_key, logger=logger)
logger.info('updated_report_id={}, report_id={}'.format(updated_report_id, report_id))
if not updated_report_id or updated_report_id < report_id:
try:
p.multi()
is_updated_stop_time = _try_add_report(tracker_id, report)
p.set(updated_report_id_key, report_id)
p.execute()
done = True
except WatchError:
logger.info('WatchError: {}'.format(WatchError))
done = False
p.unwatch()
else:
logger.info('Report not used because of report_id.')
done = True
p.unwatch()
logger.info('is_updated_stop_time={}, done={}'.format(is_updated_stop_time, done))
return is_updated_stop_time and done
def track_mock_reports(self, reports, tracker_id):
for i, report in enumerate(reports):
add_report(report)
trip_delays_ids_list_of_lists = load_by_key(
get_train_tracker_trip_delays_ids_list_of_lists_key(tracker_id))
trips = get_trusted_trips(trip_delays_ids_list_of_lists)
return trips
def get_current(self):
data = load_by_key(get_train_tracker_data_key(self.tracker_id))
if data:
(state, stop_id, timestamp) = data
timestamp = ot_utils.isoformat_to_localtime(timestamp)
if stop_id == str(None):
stop_id = None
else:
stop_id = int(stop_id)
return state, stop_id, timestamp
else:
return DetectorState.states.INITIAL, None, None
def get_device_status_for_app(report):
"""
:param report
:return: trusted trip for this report
currently it just returns the first one, but later it should changed to return based on the report timestamp
"""
device_id = report.device_id
trip_delays_ids_list_of_lists = load_by_key(get_train_tracker_trip_delays_ids_list_of_lists_key(device_id))
trusted_trips = get_trusted_trips(trip_delays_ids_list_of_lists)
if trusted_trips:
return trusted_trips[0]
return None
def track_device(device_id,
do_print=False,
do_preload_reports=True,
set_reports_to_same_weekday_last_week=False,
report_limit=10000000):
#device_coords, device_timestamps, device_accuracies_in_meters, device_accuracies_in_coords = get_location_info_from_device_id(device_id)
now = ot_utils.get_localtime_now()
reports_queryset = stop_detector_test.get_device_id_reports(device_id)
assert reports_queryset.count() > 0, 'No device reports in db'
tracker_id = device_id
fps_period_start = time.clock()
fps_period_length = 100
if do_preload_reports:
reports_queryset = list(reports_queryset)
count = len(reports_queryset) if isinstance(
reports_queryset, list) else reports_queryset.count()
for i in xrange(count):
if i > report_limit:
break
if i % fps_period_length == 0:
elapsed = (time.clock() - fps_period_start)
if elapsed > 0:
logger.debug('%d\t%.1f qps' % (i, fps_period_length / elapsed))
else:
logger.debug(
'Elapsed time should be positive but is %d' % (elapsed))
fps_period_start = time.clock()
report = reports_queryset[i]
if set_reports_to_same_weekday_last_week:
# fix finding same weekday last week by http://stackoverflow.com/questions/6172782/find-the-friday-of-previous-last-week-in-python
day_fix = (now.weekday() - report.timestamp.weekday()) % 7
day = now + datetime.timedelta(days=-day_fix)
# move day and correct for DST (daylight savings time)
dst_before = report.get_timestamp_israel_time().dst()
report.timestamp = report.timestamp.replace(
year=day.year, month=day.month, day=day.day)
dst_after = report.get_timestamp_israel_time().dst()
report.timestamp -= dst_after - dst_before
add_report(report)
#tracker.print_tracked_stop_times()
#tracker.print_possible_trips()
trip_delays_ids_list_of_lists = load_by_key(
get_train_tracker_trip_delays_ids_list_of_lists_key(tracker_id))
trips = get_trusted_trips(trip_delays_ids_list_of_lists)
return tracker_id, trips
def try_update_coords(report, tracker_id):
loc = report.get_my_loc()
coords = [loc.lat, loc.lon]
res_shape_sampled_point_ids, _ = shapes.all_shapes.query_sampled_points(coords, loc.accuracy_in_coords)
added_count = cl.sadd(get_train_tracker_visited_shape_sampled_point_ids_key(tracker_id), res_shape_sampled_point_ids)
trips = load_by_key(get_train_tracker_trip_ids_key(tracker_id))
trip = trips[0] if trips else None
if added_count > 0:
save_by_key(get_train_tracker_coords_key(tracker_id), coords, cl=p)
if trip is not None and len(trips) <= TRACKER_REPORT_FOR_TRIP_COUNT_LOWER_THAN:
save_by_key(get_current_trip_id_coords_key(trip), coords, timeout=TRACKER_TTL, cl=p)
save_by_key(get_current_trip_id_coords_timestamp_key(trip), ot_utils.dt_time_to_unix_time(report.timestamp), timeout=TRACKER_TTL, cl=p)
p.execute()
if trip is not None:
cl.setex(get_current_trip_id_report_timestamp_key(trip), TRACKER_TTL, ot_utils.dt_time_to_unix_time(report.timestamp))
def GetRedisData(redis_key, day=None):
if day:
redis_key += ':' + _DayToDayStr(day)
if redis_key in redis_cache:
return redis_cache[redis_key]
data_json = load_by_key(redis_key)
data = json.loads(data_json,object_hook=json_hook_dt)
import dateutil.parser
if type(data) == dict:
for trip_id in data.keys():
if type(data[trip_id]) == dict and 'start_time' in data[trip_id].keys():
data[trip_id]['start_time'] = ot_utils.get_localtime(dateutil.parser.parse(data[trip_id]['start_time']))
data[trip_id]['end_time'] = ot_utils.get_localtime(dateutil.parser.parse(data[trip_id]['end_time']))
for stop_id in data[trip_id]['stops']:
stop_data = data[trip_id]['stops'][stop_id]
#stop_data[1] = ot_utils.get_localtime(dateutil.parser.parse(stop_data[1]))
#stop_data[2] = ot_utils.get_localtime(dateutil.parser.parse(stop_data[2]))
data[trip_id]['stops'][stop_id] = stop_data
redis_cache[redis_key] = data
return data
def update_coords(report, tracker_id):
loc = report.get_my_loc()
coords = [loc.lat, loc.lon]
logger.info('Updating coords for tracker_id={} by report={} to coords={}'.format(tracker_id, report, coords))
res_shape_sampled_point_ids, _ = shapes.all_shapes.query_sampled_points(coords, loc.accuracy_in_coords)
added_count = cl.sadd(get_train_tracker_visited_shape_sampled_point_ids_key(tracker_id), res_shape_sampled_point_ids)
trip_delays_ids_list_of_lists = load_by_key(get_train_tracker_trip_delays_ids_list_of_lists_key(tracker_id))
trips = get_trusted_trips(trip_delays_ids_list_of_lists)
for trip in trips:
cl.setex(get_current_trip_id_report_timestamp_key(trip), TRACKER_TTL, ot_utils.dt_time_to_unix_time(report.timestamp))
if added_count > 0:
save_by_key(get_current_trip_id_coords_key(trip),\
coords,\
timeout=TRACKER_TTL, cl=p)
save_by_key(get_current_trip_id_coords_timestamp_key(trip),\
ot_utils.dt_time_to_unix_time(report.timestamp),\
timeout=TRACKER_TTL, cl=p)
else: # extend expiration:
p.expire(get_current_trip_id_coords_key(trip), TRACKER_TTL)
p.expire(get_current_trip_id_coords_timestamp_key(trip), TRACKER_TTL)
p.execute()
def get_current_location(trip):
from redis_intf.client import load_by_key
return load_by_key('current_trip_id:coords:%s' % (trip.gtfs_trip_id))
def get_possible_trips(tracker_id, print_debug_info=False):
stop_times_redis = cl.zrange(get_train_tracker_tracked_stops_key(tracker_id), 0, -1, withscores=True)
if len(stop_times_redis) == 0:
return None
#arrival = get_localtime(datetime.datetime.fromtimestamp(stop_times_redis[0][1]), self.db_timezone)
#shape_matches_inds = self.get_shape_ids_with_high_probability()
#trips = gtfs.models.Trip.objects.filter(shape_id__in=shape_matches_inds, service__in=relevant_service_ids)
relevant_service_ids = load_by_key(get_train_tracker_relevant_services_key(tracker_id))
trips = gtfs.models.Trip.objects.filter(service__in=relevant_service_ids)
# filter by stop existence and its time frame:
trips_filtered_by_stops_and_times = trips
if print_debug_info:
print len(trips_filtered_by_stops_and_times)
stop_times = []
for cur in stop_times_redis:
arrival = ot_utils.unix_time_to_localtime(int(cur[1]))
cur_0_split = cur[0].split('_')
stop_id = cur_0_split[0]
name = stops.all_stops[stop_id].name
departure = ot_utils.unix_time_to_localtime(int(cur_0_split[1])) if cur_0_split[1] != '' else None
stop_times.append(TrackedStopTime(stop_id))
stop_times[-1].arrival = arrival
stop_times[-1].departure = departure
trip_stop_times = gtfs.models.StopTime.objects.filter(trip__in = trips_filtered_by_stops_and_times)
arrival_time__greater_than = arrival-datetime.timedelta(seconds=config.late_arrival_max_seconds)
arrival_time__less_than = arrival+datetime.timedelta(seconds=config.early_arrival_max_seconds)
arrival_time__range = datetime_range_to_db_time(arrival_time__greater_than, arrival_time__less_than)
if departure is not None:
departure_time__greater_than = departure-datetime.timedelta(seconds=config.late_departure_max_seconds)
departure_time__less_than = departure+datetime.timedelta(seconds=config.early_departure_max_seconds)
departure_time__range = datetime_range_to_db_time(departure_time__greater_than, departure_time__less_than)
trip_stop_times_for_specific_stop = trip_stop_times.filter(stop = stop_id,
arrival_time__range=arrival_time__range,
departure_time__range=departure_time__range)
else:
trip_stop_times_for_specific_stop = trip_stop_times.filter(stop = stop_id,
arrival_time__range=arrival_time__range)
trips_filtered_by_stops_and_times = trip_stop_times_for_specific_stop.values_list('trip')
if print_debug_info:
print len(trips_filtered_by_stops_and_times)
# filter by stop order:
trip_in_right_direction = []
for i, t in enumerate(trips_filtered_by_stops_and_times):
trip_stop_times = gtfs.models.StopTime.objects.filter(trip = t).order_by('arrival_time').values_list('stop')
trip_stop_times = [str(x[0]) for x in trip_stop_times]
stop_inds_by_visit_order = [trip_stop_times.index(x) for x in [x.stop_id for x in stop_times]]
if is_increasing(stop_inds_by_visit_order):
trip_in_right_direction.append(i)
trips_filtered_by_stops_and_times = [trips_filtered_by_stops_and_times[i][0] for i in trip_in_right_direction]
arrival_delta_abs_sums_seconds = []
#departure_delta_abs_sums = []
for t in trips_filtered_by_stops_and_times:
stop_times_redis = gtfs.models.StopTime.objects.filter(trip = t).order_by('arrival_time').values_list('stop', 'arrival_time')#, 'departure_time')
arrival_delta_abs_sum = 0
#departure_delta_abs_sum = 0
for tracked_stop_time in stop_times:
stop_time = [x for x in stop_times_redis if str(x[0]) == tracked_stop_time.stop_id][0]
arrival_delta_seconds = stop_time[1] - datetime_to_db_time(tracked_stop_time.arrival)
#departure_delta_seconds = stop_time[2] - datetime_to_db_time(tracked_stop_time.departure)
arrival_delta_abs_sum += abs(arrival_delta_seconds)
#departure_delta_abs_sum += abs(departure_delta_seconds)
arrival_delta_abs_sums_seconds.append(arrival_delta_abs_sum)
#departure_delta_abs_sums.append(departure_delta_abs_sum)
# sort results by increasing arrival time
sorted_trips = sorted(zip(arrival_delta_abs_sums_seconds,trips_filtered_by_stops_and_times))
trips_filtered_by_stops_and_times = [x for (y,x) in sorted_trips]
arrival_delta_abs_sums_seconds = [y for (y,x) in sorted_trips]
return trips_filtered_by_stops_and_times, arrival_delta_abs_sums_seconds
def add_report_to_tracker(tracker_id, report):
if not load_by_key(get_train_tracker_relevant_services_key(tracker_id)):
start_date = report.timestamp.strftime("%Y-%m-%d")
relevant_services = gtfs.models.Service.objects.filter(start_date = start_date)
relevant_service_ids = [x[0] for x in relevant_services.all().values_list('service_id')]
save_by_key(get_train_tracker_relevant_services_key(tracker_id), relevant_service_ids)
# update train position
if report.get_my_loc():
try_update_coords(report, tracker_id)
##commented out below is code that filters trips based on shape
#coords_updated = False
#p = get_redis_pipeline()
#p.zincrby("train_tracker:%s:counters" % (tracker_id), res_shape_ids[i], inc_by)
#p.incr("train_tracker:%s:total" % (tracker_id), inc_by)
#p.execute()
#for i in xrange(len(res_shape_point_ids)):
#cl = get_redis_client()
#if cl.sadd("train_tracker:%s:visited_shape_point_ids" % (tracker_id), res_shape_point_ids[i]) == 0:
#if not coords_updated: #update if report adds new points on tracks
#self.coords = coords
#coords_updated = True
#p = get_redis_pipeline()
#p.zincrby("train_tracker:%s:counters" % (tracker_id), res_shape_ids[i], 1)
#p.incr("train_tracker:%s:total" % (tracker_id))
#p.execute()
##self.shape_counts[res_shape_ids[i]] = self.shape_counts[res_shape_ids[i]] + 1
# 1) add stop or non-stop to prev_stops and prev_stops_timestamps
# 2) set calc_hmm to true if according to wifis and/or location, our state changed from stop to non-stop or vice versa
prev_current_stop_id_by_hmm = cl.get(get_train_tracker_current_stop_id_key(tracker_id))
if not prev_current_stop_id_by_hmm:
prev_state = tracker_states.INITIAL
elif prev_current_stop_id_by_hmm == stops.NOSTOP:
prev_state = tracker_states.NOSTOP
else:
prev_state = tracker_states.STOP
stop_id = try_get_stop_id(report)
if not stop_id:
current_state = tracker_states.UNKNOWN
elif stop_id == nostop_id:
current_state = tracker_states.NOSTOP
else:
current_state = tracker_states.STOP
if current_state != tracker_states.UNKNOWN:
timestamp = report.get_timestamp_israel_time()
prev_stop_id = add_prev_stop(tracker_id, stop_id, timestamp)
# calculate hmm to get state_sequence, update stop_times and current_stop if needed
if current_state != tracker_states.UNKNOWN and prev_state != current_state:
prev_stops_and_timestamps = cl.zrange(get_train_tracker_timestamp_sorted_stop_ids_key(tracker_id), 0, -1, withscores=True)
prev_stop_ids_order = [int(x[0].split("_")[0]) for x in prev_stops_and_timestamps]
prev_stops_and_timestamps = [x for (y,x) in sorted(zip(prev_stop_ids_order,prev_stops_and_timestamps))]
prev_stop_ids = [x[0].split("_")[1] for x in prev_stops_and_timestamps]
prev_stop_int_ids = np.array([stops.all_stops.id_list.index(x) for x in prev_stop_ids])
#assert np.array_equal(prev_stop_int_ids, np.array(self.prev_stops))
prev_stop_hmm_logprob, prev_stop_int_ids_by_hmm = hmm.decode(prev_stop_int_ids)
prev_stop_int_ids_by_hmm_for_debug = prev_stop_int_ids_by_hmm
# update current_stop_id_by_hmm and current_state by hmm:
current_stop_id_by_hmm = stops.all_stops.id_list[prev_stop_int_ids_by_hmm[-1]]
cl.set(get_train_tracker_current_stop_id_key(tracker_id), current_stop_id_by_hmm)
if current_stop_id_by_hmm == stops.NOSTOP:
current_state = tracker_states.NOSTOP
else:
current_state = tracker_states.STOP
if prev_state != current_state: # change in state
prev_stops_by_hmm = [stops.all_stops.id_list[x] for x in prev_stop_int_ids_by_hmm]
prev_stops_timestamps = [ot_utils.unix_time_to_localtime((x[1])) for x in prev_stops_and_timestamps]
index_of_oldest_current_state = max(0, find_index_of_first_consecutive_value(prev_stops_by_hmm, len(prev_stops_by_hmm)-1))
index_of_most_recent_previous_state = index_of_oldest_current_state-1
if current_state == tracker_states.NOSTOP:
stop_id = prev_stops_by_hmm[index_of_most_recent_previous_state]
unix_timestamp = ot_utils.dt_time_to_unix_time(prev_stops_timestamps[index_of_most_recent_previous_state])
if prev_state == tracker_states.INITIAL:
pass #do nothing
else: # previous_state == tracker_states.STOP - need to set stop_time departure
stop_time = cl.zrange(get_train_tracker_tracked_stops_key(tracker_id), -1, -1, withscores=True)
departure_unix_timestamp = unix_timestamp
stop_id_and_departure_time = "%s_%d" % (prev_current_stop_id_by_hmm, departure_unix_timestamp)
update_stop_time(tracker_id, prev_stop_id, stop_time[0][1], stop_id_and_departure_time)
update_trips(tracker_id)
else: # current_state == tracker_states.STOP
stop_id = prev_stops_by_hmm[index_of_oldest_current_state]
unix_timestamp = ot_utils.dt_time_to_unix_time(prev_stops_timestamps[index_of_oldest_current_state])
arrival_unix_timestamp = unix_timestamp
stop_id_and_departure_time = "%s_" % (current_stop_id_by_hmm)
update_stop_time(tracker_id, prev_stop_id, arrival_unix_timestamp, stop_id_and_departure_time)
update_trips(tracker_id)
prev_timestamp = unix_timestamp
print_tracked_stop_times(tracker_id)
def get_device_status(device_id):
trip_delays_ids_list_of_lists = load_by_key(get_train_tracker_trip_delays_ids_list_of_lists_key(device_id))
return trip_delays_ids_list_of_lists
def get_trips(tracker_id):
trips = load_by_key(get_train_tracker_trip_ids_key(tracker_id))
time_deviation_in_seconds = load_by_key(get_train_tracker_trip_ids_deviation_seconds_key(tracker_id))
return trips, time_deviation_in_seconds
