def __init__(self, name, soma_map, soma_config, minute_interval, window_interval, collection="occurrence_rates"): self.topo_map = None self.soma_map = soma_map self.soma_config = soma_config self.minute_interval = minute_interval self.window_interval = window_interval self.rotm = RegionObservationTimeManager(soma_map, soma_config) self.tre = TrajectoryRegionEstimate(soma_map, soma_config, minute_interval) self.tof = TrajectoryOccurrenceFrequencies( soma_map, soma_config, minute_interval=minute_interval, window_interval=window_interval) self.gs = GeoSpatialStoreProxy('geospatial_store', 'soma') rospy.loginfo("Connect to database collection %s..." % collection) self._ms = MessageStoreProxy(collection=collection) self._soma_db = MessageStoreProxy(collection="soma_roi") rospy.loginfo("Create a service %s/service..." % name) self.service = rospy.Service(name + '/service', TrajectoryOccurrenceRate, self.srv_cb) rospy.loginfo("Create an action server %s..." % name) self._as = actionlib.SimpleActionServer(name, OccurrenceRateLearningAction, execute_cb=self.execute, auto_start=False) self._as.start()
def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10): """ Initialise analysis for trajectory periodicity. Initialise trajectory occurrence frequencies from database. """ self.tof = TrajectoryOccurrenceFrequencies(soma_map, soma_config, minute_interval, window_interval) self.periodic_type = self.tof.periodic_type self.length_of_periodicity = len(self.tof.periodic_days) self.minute_interval = minute_interval self.window_interval = window_interval self.tof.load_tof() self.tof = self.tof.tof self.regions = self.tof.keys() self.spectrum_selection = 0 self.addition_technique = True
def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10): """ Initialize plotting, it needs the name of the soma map, specific soma configuration, the minute interval between two different occurrence rate, and the window interval that acts as bins. """ self.tof = TrajectoryOccurrenceFrequencies(soma_map, soma_config, minute_interval, window_interval) self.xticks = time_ticks(minute_interval, window_interval, self.tof.periodic_type) self.x = np.arange(len(self.xticks)) self.minute_interval = minute_interval self.window_interval = window_interval self.periodic_length = len(self.tof.periodic_days) self.tof.load_tof() self.tof = self.tof.tof self.regions = self.tof.keys() self.colors = [ (0., 0., 0.), (0., 0., 1.), (0., 1., 0.), (0., 1., 1.), (1., 0., 0.), (1., 0., 1.), (1., 1., 0.), (.75, .75, .75), (0., 0., 0.), (0., 0., .5), (0., .5, 0.), (0., .5, .5), (.5, 0., 0.), (.5, 0., .5), (.5, .5, 0.), (.25, .25, .25) ]
def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10): """ Initialise analysis for trajectory periodicity. Initialise trajectory occurrence frequencies from database. """ self.tof = TrajectoryOccurrenceFrequencies( soma_map, soma_config, minute_interval, window_interval ) self.periodic_type = self.tof.periodic_type self.length_of_periodicity = len(self.tof.periodic_days) self.minute_interval = minute_interval self.window_interval = window_interval self.tof.load_tof() self.tof = self.tof.tof self.regions = self.tof.keys() self.spectrum_selection = 0 self.addition_technique = True
def __init__(self, name, soma_map, soma_config, minute_interval, window_interval, collection="occurrence_rates"): self.soma_map = soma_map self.soma_config = soma_config self.minute_interval = minute_interval self.window_interval = window_interval self.rotm = RegionObservationTimeManager(soma_map, soma_config) self.tre = TrajectoryRegionEstimate(soma_map, soma_config, minute_interval) self.tof = TrajectoryOccurrenceFrequencies( soma_map, soma_config, minute_interval=minute_interval, window_interval=window_interval ) rospy.loginfo("Connect to database collection %s..." % collection) self._ms = MessageStoreProxy(collection=collection) rospy.loginfo("Create a service %s/service..." % name) self.service = rospy.Service(name + "/service", TrajectoryOccurrenceRate, self.srv_cb) rospy.loginfo("Create an action server %s..." % name) self._as = actionlib.SimpleActionServer( name, OccurrenceRateLearningAction, execute_cb=self.execute, auto_start=False ) self._as.start()
if __name__ == '__main__': rospy.init_node("synthetic_tof") if len(sys.argv) < 4: rospy.logerr("usage: synthetic minute_interval window_interval store(0)/test(1)") sys.exit(2) interval = int(sys.argv[1]) window = int(sys.argv[2]) sw = SyntheticWave(interval) if not int(sys.argv[3]): waves = sw.get_one_month_synthetic_wave(True) tof = TOF("synthetic", "synthetic_config", interval, window) tof.load_tof() for i in range(4, 31+1): prev_traj_est = trajectory_estimate_for_date( waves, datetime.date(2015, 5, i-1) ) curr_traj_est = trajectory_estimate_for_date( waves, datetime.date(2015, 5, i) ) tof.update_tof_daily( curr_traj_est, prev_traj_est, datetime.date(2015, 5, i) ) tof.store_tof() else: waves = sw.get_one_week_synthetic_wave(True) tp = TrajectoryPeriodicity("synthetic", "synthetic_config", interval, window)
class TrajectoryArrivalRateManager(object): def __init__(self, name, soma_map, soma_config, minute_interval, window_interval, collection="occurrence_rates"): self.topo_map = None self.soma_map = soma_map self.soma_config = soma_config self.minute_interval = minute_interval self.window_interval = window_interval self.rotm = RegionObservationTimeManager(soma_map, soma_config) self.tre = TrajectoryRegionEstimate(soma_map, soma_config, minute_interval) self.tof = TrajectoryOccurrenceFrequencies( soma_map, soma_config, minute_interval=minute_interval, window_interval=window_interval) self.gs = GeoSpatialStoreProxy('geospatial_store', 'soma') rospy.loginfo("Connect to database collection %s..." % collection) self._ms = MessageStoreProxy(collection=collection) self._soma_db = MessageStoreProxy(collection="soma_roi") rospy.loginfo("Create a service %s/service..." % name) self.service = rospy.Service(name + '/service', TrajectoryOccurrenceRate, self.srv_cb) rospy.loginfo("Create an action server %s..." % name) self._as = actionlib.SimpleActionServer(name, OccurrenceRateLearningAction, execute_cb=self.execute, auto_start=False) self._as.start() def execute(self, goal): temp_start_time = rospy.Time.now() curr_date = datetime.datetime.fromtimestamp(rospy.Time.now().secs) # curr_date = datetime.datetime(2016, 3, 10, 0, 50) if curr_date.hour >= 0 and curr_date.hour < 8: curr_date = curr_date - datetime.timedelta(hours=24) curr_date = datetime.datetime(curr_date.year, curr_date.month, curr_date.day, 0, 0) prev_date = curr_date - datetime.timedelta(hours=24) self.rotm.calculate_region_observation_duration([prev_date, curr_date], self.minute_interval) self.rotm.store_to_mongo() if self._as.is_preempt_requested(): self._as.set_preempted() return curr_traj_est = self.tre.estimate_trajectories_daily([curr_date.day], curr_date.month, curr_date.year) curr_traj_est = trajectory_estimate_for_date(curr_traj_est, curr_date) prev_traj_est = self.tre.estimate_trajectories_daily([prev_date.day], prev_date.month, prev_date.year) prev_traj_est = trajectory_estimate_for_date(prev_traj_est, prev_date) if self._as.is_preempt_requested(): self._as.set_preempted() return self.tof.update_tof_daily(curr_traj_est, prev_traj_est, curr_date) self.tof.store_tof() self.rotm.reinit() self.tof.reinit() temp_end_time = rospy.Time.now() rospy.loginfo("Time needed to complete this is %d" % (temp_end_time - temp_start_time).secs) self._as.set_succeeded(OccurrenceRateLearningResult()) def _get_occurrence_rate_logs(self, start_time, end_time): filtered_logs = list() if (end_time - start_time).total_seconds() <= self.window_interval * 60: end_time = start_time + datetime.timedelta( seconds=self.minute_interval * 60) if start_time.hour == end_time.hour and start_time.day == end_time.day: query = { "soma": self.soma_map, "soma_config": self.soma_config, "duration.secs": self.window_interval * 60, "day": start_time.weekday(), "hour": start_time.hour, "minute": { "$gte": start_time.minute, "$lt": end_time.minute } } logs = self._ms.query(OccurrenceRate._type, message_query=query) filtered_logs = logs elif start_time.hour != end_time.hour and start_time.day == end_time.day: query = { "soma": self.soma_map, "soma_config": self.soma_config, "duration.secs": self.window_interval * 60, "day": start_time.weekday(), "hour": { "$gte": start_time.hour, "$lte": end_time.hour }, } logs = self._ms.query(OccurrenceRate._type, message_query=query) for log in logs: if log[0].hour == start_time.hour and log[ 0].minute >= start_time.minute: filtered_logs.append(log) elif log[0].hour == end_time.hour and log[ 0].minute < end_time.minute: filtered_logs.append(log) elif log[0].hour > start_time.hour and log[ 0].hour < end_time.hour: filtered_logs.append(log) else: day = start_time.weekday() end_day = end_time.weekday() query = { "soma": self.soma_map, "soma_config": self.soma_config, "duration.secs": self.window_interval * 60, "day": { "$gte": day, "$lte": end_day } } logs = self._ms.query(OccurrenceRate._type, message_query=query) for log in logs: if log[0].day == day and log[0].hour >= start_time.hour and log[ 0].minute >= start_time.minute: filtered_logs.append(log) if log[0].day == end_day and log[ 0].hour <= end_time.hour and log[ 0].minute < end_time.minute: filtered_logs.append(log) elif end_day > day: if log[0].day > day and log[0].day < end_day: filtered_logs.append(log) elif end_day < day: if log[0].day > day: filtered_logs.append(log) elif log[0].day < end_day: filtered_logs.append(log) return filtered_logs def _choose_proper_region_to_observe(self, rois, wp_point): accumulate_dist = dict() for roi in rois: query = { "map": self.soma_map, "config": self.soma_config, "roi_id": roi } logs = self._soma_db.query(SOMAROIObject._type, message_query=query) for log in logs: if roi not in accumulate_dist: accumulate_dist[roi] = 0.0 accumulate_dist[roi] = distance_calc.euclidean( [wp_point.pose.position.x, wp_point.pose.position.y], [log[0].pose.position.x, log[0].pose.position.y]) accumulate_dist = sorted(accumulate_dist, key=lambda i: accumulate_dist[i], reverse=True) return accumulate_dist[-1] def _get_waypoints(self, region_ids): waypoints = dict() topo_sub = rospy.Subscriber("/topological_map", TopologicalMap, self._topo_map_cb, None, 10) while self.topo_map is None: rospy.sleep(0.1) rospy.logwarn("Trying to get information from /topological_map...") topo_sub.unregister() for wp in self.topo_map.nodes: if wp.name == "ChargingPoint": continue _, _, yaw = euler_from_quaternion( [0, 0, wp.pose.orientation.z, wp.pose.orientation.w]) coords = robot_view_cone(wp.pose.position.x, wp.pose.position.y, yaw) langitude_latitude = list() for pt in coords: langitude_latitude.append( self.gs.coords_to_lnglat(pt[0], pt[1])) langitude_latitude.append( self.gs.coords_to_lnglat(coords[0][0], coords[0][1])) if self.gs.observed_roi(langitude_latitude, self.soma_map, self.soma_config).count() > 1: rospy.logwarn( "There are two or more regions covered by the sight of the robot in %s" % wp.name) roi = list() for j in self.gs.observed_roi(langitude_latitude, self.soma_map, self.soma_config): roi.append(str(j['soma_roi_id'])) roi = [i for i in roi if i in region_ids] if len(roi) < 1: continue elif len(roi) == 1: waypoints[roi[0]] = wp.name else: roi = self._choose_proper_region_to_observe(roi, wp) waypoints[roi] = wp.name return waypoints def _topo_map_cb(self, topo_map): self.topo_map = topo_map def _get_long_duration(self, four_tuple, start_time): datetimes = list() for i in four_tuple: delta_day = (i[0] - start_time.weekday()) % 7 curr_date = start_time + datetime.timedelta(days=delta_day) curr_date = datetime.datetime(curr_date.year, curr_date.month, curr_date.day, i[1], i[2]) datetimes.append(curr_date) datetimes = sorted(datetimes) counter = 0 for i in range(1, len(datetimes)): if (datetimes[i] - datetimes[i - 1]).seconds == self.minute_interval * 60: counter += 1 elif (datetimes[i] - datetimes[i - 1]).seconds <= self.window_interval * 60: counter += (datetimes[i] - datetimes[i - 1]).seconds / ( self.minute_interval * 60) duration = rospy.Duration(self.window_interval * 60 + (counter * self.minute_interval * 60)) best_time = rospy.Time(time.mktime(datetimes[0].timetuple())) return best_time, duration def _get_best_regions(self, logs, start_time): or_regions = dict() best_time = dict() for log in logs: if log[0].region_id not in or_regions: or_regions[log[0].region_id] = 0.0 or_regions[log[0].region_id] += log[0].occurrence_rate if log[0].region_id not in best_time: best_time[log[0].region_id] = [[ log[0].day, log[0].hour, log[0].minute, log[0].occurrence_rate ]] elif best_time[log[0].region_id][-1][-1] < log[0].occurrence_rate: best_time[log[0].region_id][-1] = [ log[0].day, log[0].hour, log[0].minute, log[0].occurrence_rate ] elif best_time[log[0].region_id][-1][-1] == log[0].occurrence_rate: best_time[log[0].region_id].append([ log[0].day, log[0].hour, log[0].minute, log[0].occurrence_rate ]) duration = dict() for roi in best_time.keys(): temp = best_time[roi] if len(temp) > 0: best_time[roi], duration[roi] = self._get_long_duration( temp, start_time) else: rospy.logwarn( "There is no recommended visiting time for region %s" % roi) duration[roi] = rospy.Duration(0) best_time[roi] = rospy.Time.now() return or_regions, best_time, duration def srv_cb(self, msg): region_ids = list() best_times_list = list() durations_list = list() waypoints_list = list() rospy.loginfo("Got a request...") print msg rospy.loginfo( "Retrieving trajectory occurrence frequencies from database...") start_time = datetime.datetime.fromtimestamp(msg.start_time.secs) end_time = datetime.datetime.fromtimestamp(msg.end_time.secs) if end_time >= start_time: logs = self._get_occurrence_rate_logs(start_time, end_time) # calculate occurrence_rate per region or_regions, best_times, durations = self._get_best_regions( logs, start_time) region_ids = sorted(or_regions, key=lambda i: or_regions[i], reverse=True) # get the nearest waypoints somehow waypoints = self._get_waypoints(region_ids) # ordering for roi in region_ids: best_times_list.append(best_times[roi]) durations_list.append(durations[roi]) waypoints_list.append(waypoints[roi]) rospy.loginfo("Composing answer...") print TrajectoryOccurrenceRateResponse( region_ids[:msg.n_best], best_times_list[:msg.n_best], durations_list[:msg.n_best], waypoints_list[:msg.n_best]) return TrajectoryOccurrenceRateResponse(region_ids[:msg.n_best], best_times_list[:msg.n_best], durations_list[:msg.n_best], waypoints_list[:msg.n_best])
rospy.init_node("synthetic_tof") if len(sys.argv) < 4: rospy.logerr( "usage: synthetic minute_interval window_interval store(0)/test(1)" ) sys.exit(2) interval = int(sys.argv[1]) window = int(sys.argv[2]) sw = SyntheticWave(interval) if not int(sys.argv[3]): waves = sw.get_one_month_synthetic_wave(True) # waves = sw.get_one_month_synthetic_wave(False) # no noise tof = TOF("synthetic", "synthetic_config", interval, window) tof.load_tof() for i in range(4, 31 + 1): prev_traj_est = trajectory_estimate_for_date( waves, datetime.date(2015, 5, i - 1)) curr_traj_est = trajectory_estimate_for_date( waves, datetime.date(2015, 5, i)) tof.update_tof_daily(curr_traj_est, prev_traj_est, datetime.date(2015, 5, i)) tof.store_tof() else: waves = sw.get_one_week_synthetic_wave(True) tp = TrajectoryPeriodicity("synthetic", "synthetic_config", interval, window) # comment from here if just want the plot inp = raw_input("MSE(0) or Prediction MSE(1): ")
class TrajectoryOccurrenceRateManager(object): def __init__(self, name, soma_map, soma_config, minute_interval, window_interval, collection="occurrence_rates"): self.soma_map = soma_map self.soma_config = soma_config self.minute_interval = minute_interval self.window_interval = window_interval self.rotm = RegionObservationTimeManager(soma_map, soma_config) self.tre = TrajectoryRegionEstimate(soma_map, soma_config, minute_interval) self.tof = TrajectoryOccurrenceFrequencies( soma_map, soma_config, minute_interval=minute_interval, window_interval=window_interval ) rospy.loginfo("Connect to database collection %s..." % collection) self._ms = MessageStoreProxy(collection=collection) rospy.loginfo("Create a service %s/service..." % name) self.service = rospy.Service(name + "/service", TrajectoryOccurrenceRate, self.srv_cb) rospy.loginfo("Create an action server %s..." % name) self._as = actionlib.SimpleActionServer( name, OccurrenceRateLearningAction, execute_cb=self.execute, auto_start=False ) self._as.start() def execute(self, goal): temp_start_time = rospy.Time.now() curr_date = datetime.datetime.fromtimestamp(rospy.Time.now().secs) # curr_date = datetime.datetime(2016, 3, 10, 0, 50) if curr_date.hour >= 0 and curr_date.hour < 8: curr_date = curr_date - datetime.timedelta(hours=24) curr_date = datetime.datetime(curr_date.year, curr_date.month, curr_date.day, 0, 0) prev_date = curr_date - datetime.timedelta(hours=24) self.rotm.calculate_region_observation_duration([prev_date, curr_date], self.minute_interval) self.rotm.store_to_mongo() if self._as.is_preempt_requested(): self._as.set_preempted() return curr_traj_est = self.tre.estimate_trajectories_daily([curr_date.day], curr_date.month, curr_date.year) curr_traj_est = trajectory_estimate_for_date(curr_traj_est, curr_date) prev_traj_est = self.tre.estimate_trajectories_daily([prev_date.day], prev_date.month, prev_date.year) prev_traj_est = trajectory_estimate_for_date(prev_traj_est, prev_date) if self._as.is_preempt_requested(): self._as.set_preempted() return self.tof.update_tof_daily(curr_traj_est, prev_traj_est, curr_date) self.tof.store_tof() self.rotm.reinit() self.tof.reinit() temp_end_time = rospy.Time.now() rospy.loginfo("Time needed to complete this is %d" % (temp_end_time - temp_start_time).secs) self._as.set_succeeded(OccurrenceRateLearningResult()) def srv_cb(self, msg): rospy.loginfo("Retrieving trajectory occurrence frequencies from database...") print msg query = { "soma": self.soma_map, "soma_config": self.soma_config, "region_id": msg.region_id, "day": msg.day, "end_hour": msg.hour, "end_minute": msg.minute, } logs = self._ms.query(OccurrenceRate._type, message_query=query) occurrence_rate = 0.0 counter = 0 for log in logs: occurrence_rate += log[0].occurrence_rate counter += 1 if counter == 0: counter += 1 if len(logs) > 0: rospy.loginfo("Occurrence Rate is obtained, which has value %.2f" % (occurrence_rate / float(counter))) else: rospy.loginfo("No occurrence rate is obtained, returning zero value") return TrajectoryOccurrenceRateResponse(occurrence_rate / float(counter))
class TrajectoryPeriodicity(object): def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10): """ Initialise analysis for trajectory periodicity. Initialise trajectory occurrence frequencies from database. """ self.tof = TrajectoryOccurrenceFrequencies(soma_map, soma_config, minute_interval, window_interval) self.periodic_type = self.tof.periodic_type self.length_of_periodicity = len(self.tof.periodic_days) self.minute_interval = minute_interval self.window_interval = window_interval self.tof.load_tof() self.tof = self.tof.tof self.regions = self.tof.keys() self.spectrum_selection = 0 self.addition_technique = True def get_tof_values(self, region): """ Obtain trajectory occurrence frequency values for a specific region. """ length = (self.window_interval / self.minute_interval) - 1 y = list() region_tof = self.tof[region] for day, hourly_tof in region_tof.iteritems(): mins = sorted(hourly_tof) daily_y = list() for i in mins: daily_y.append(hourly_tof[i].get_occurrence_rate()) y.extend(daily_y[-length:] + daily_y[:-length]) return y def reconstruct_tof_from_spectrum(self, region, addition_method=True, num_of_freqs=30): """ Reconstruct trajectory occurrence frequency values after they are transformed into spectral dimensions. """ original = self.get_tof_values(region) if addition_method: spectrums, _ = self.get_significant_frequencies( original, num_of_freqs * 2) spectrums = spectrums[0:num_of_freqs] else: spectrums = self.get_highest_n_freq(fft(original), num_of_freqs) reconstruction = 0 for spectrum in spectrums: reconstruction += self.rectify_wave(spectrum[2], spectrum[0], spectrum[1], len(original)) reconstruction = map( lambda x: Lambda().get_occurrence_rate() if x <= 0.0 else x, reconstruction) return reconstruction, original def model_selection(self, data, month, year, addition_method=True, max_freqs=30): spectrum_selection = list() validate_data = dict() for region in self.regions: validate_data.update({ region: trajectories_full_dates_periodic(data[region], month, year, self.length_of_periodicity, self.window_interval, self.minute_interval) }) for num_of_freqs in range(1, max_freqs + 1): mse_region = list() for region in self.regions: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, addition_method, num_of_freqs) mse_recon, _ = self._calculate_mse(reconstruction_tof, original_tof, validate_data[region]) mse_region.append(mse_recon) mse_region = [i for i in mse_region if i != -1] spectrum_selection.append(sum(mse_region) / float(len(mse_region))) spectrum_selection = spectrum_selection.index( min(spectrum_selection)) + 1 rospy.loginfo("Ideal total spectrums: %d" % spectrum_selection) self.spectrum_selection = spectrum_selection self.addition_technique = addition_method def calculate_mse(self, region, data, month, year): if self.spectrum_selection != 0: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, self.addition_technique, self.spectrum_selection) else: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, self.addition_technique) test_data = trajectories_full_dates_periodic( data, month, year, self.length_of_periodicity, self.window_interval, self.minute_interval) reconstruction_tof = [ i for j, i in enumerate(reconstruction_tof) if j % 5 == 0 ] original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0] test_data = [i for j, i in enumerate(test_data) if j % 5 == 0] # Dropping -1 in test_data together with corresponding tof test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data( test_data, reconstruction_tof, original_tof) mse_recon = -1 mse_origin = -1 if len(reconstruction_tof) > 0 and len(original_tof) > 0: mse_recon = mse(test_data, reconstruction_tof) mse_origin = mse(test_data, original_tof) rospy.loginfo("Calculated MSE for original tof Region %s: %.2f" % (region, mse_origin)) rospy.loginfo("Calculated MSE for reconstruction tof Region %s: %.2f" % (region, mse_recon)) # temp_recon = np.sqrt(mse_recon) # sum_recon = 0 # for i in test_data: # sum_recon += (i - temp_recon)**2 # print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1))) # temp_recon = np.sqrt(mse_origin) # sum_recon = 0 # for i in test_data: # sum_recon += (i - temp_recon)**2 # print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1))) return mse_recon, mse_origin def rectify_wave(self, freq, amp, phs, num_of_points, up_thres=None, low_thres=None): xf = np.linspace(0.0, num_of_points, num_of_points) # frequency varations wave = amp * np.cos((freq * 2.0 * np.pi * xf) + phs) for ind, val in enumerate(wave): if low_thres is not None and val < low_thres: wave[ind] = low_thres if up_thres is not None and val > up_thres: wave[ind] = up_thres return wave def get_significant_frequencies(self, data, total_freq=15, max_addition=10, max_iteration=1000): N = len(data) xf = np.linspace(0.0, N, N) # initialise significant frequencies by taking frequency 0 spectrum_data = fft(data) [amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 1)[0] frequencies = [[amp, phs, freq]] freq_occur_counter = {freq: 1} exit_counter = 0 # data -= amp while len(frequencies) < total_freq: spectrum_data = fft(data) # recreate wave of the highest frequency [amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 2)[1] if freq == 0: [amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 2)[0] wave = amp * np.cos((freq * 2.0 * np.pi * xf) + phs) # substracting data with the wave data -= wave if freq not in zip(*frequencies)[2]: frequencies.append([amp, phs, freq]) freq_occur_counter.update({freq: 1}) else: for ind, val in enumerate(frequencies): if frequencies[ind][2] == freq and freq_occur_counter[ freq] < max_addition: frequencies[ind][0] += amp frequencies[ind][1] = ( (freq_occur_counter[freq] * frequencies[ind][1]) + phs) / (freq_occur_counter[freq] + 1) freq_occur_counter[freq] += 1 exit_counter += 1 if exit_counter >= max_iteration: break return frequencies, data def get_highest_n_freq(self, freqs, n=15): N = len(freqs) freqs = freqs[0:N / 2] indices = [i for i in range(len(freqs))] angles = np.angle(freqs) amplitudes = np.abs(freqs) / float(N) sorted_result = sorted(zip(amplitudes, angles, indices), reverse=True) n_freqs = sorted_result[:n] return n_freqs def _calculate_mse(self, reconstruction_tof, original_tof, test_data): # reconstruction_tof = [i for j, i in enumerate(reconstruction_tof) if j % 5 == 0] # original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0] # test_data = [i for j, i in enumerate(test_data) if j % 5 == 0] # Dropping -1 in test_data together with corresponding tof test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data( test_data, reconstruction_tof, original_tof) mse_recon = -1 mse_origin = -1 if len(reconstruction_tof) > 0 and len(original_tof) > 0: mse_recon = mse(test_data, reconstruction_tof) mse_origin = mse(test_data, original_tof) return mse_recon, mse_origin def _remove_unobserved_data(self, data1, data2, data3): # Dropping -1 in data1, the corresponding indices in data2 and data3 are # also dropped deleted_indices = list() for ind, trajs in enumerate(data1): if trajs == -1: deleted_indices.append(ind) data1 = [j for i, j in enumerate(data1) if i not in deleted_indices] data2 = [j for i, j in enumerate(data2) if i not in deleted_indices] data3 = [j for i, j in enumerate(data3) if i not in deleted_indices] return data1, data2, data3, deleted_indices def prediction_accuracy(self, region, data, month, year, percentile=0.1, plot=False): if self.spectrum_selection != 0: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, self.addition_technique, self.spectrum_selection) else: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, self.addition_technique # region, False, 26 ) test_data = trajectories_full_dates_periodic( data, month, year, self.length_of_periodicity, self.window_interval, self.minute_interval) original_predict = self._get_prediction(original_tof, test_data, percentile) reconstr_predict = self._get_prediction(reconstruction_tof, test_data, percentile) _, clean_ori_pred, clean_recon_pred, indices = self._remove_unobserved_data( test_data, reconstr_predict, original_predict) if len(clean_ori_pred) and len(clean_recon_pred): mse_predict = mse(clean_ori_pred, clean_recon_pred) else: mse_predict = -1 rospy.loginfo( "Calculated MSE for prediction between original and reconstruction for Region %s: %.2f" % (region, mse_predict)) if plot: for index in indices: original_predict[index] = -1 reconstr_predict[index] = -1 x = np.linspace(0, len(test_data), len(test_data)) xticks = time_ticks(self.minute_interval, self.window_interval, self.periodic_type) plt.plot(x, original_predict, "-o", label="Prediction Original TOF") plt.plot(x, reconstr_predict, "-^", label="Prediction Reconstruction TOF") plt.title("Prediction for Region %s" % region) plt.xticks(x, xticks, rotation="vertical") plt.xlabel( "One Week Period with %d minutes interval and %d window time" % (self.minute_interval, self.window_interval)) plt.ylabel("Prediction (1=Anomalous, 0=Normal, -1=Unobserved)") plt.ylim(ymin=-2, ymax=2) plt.legend() plt.show() def _get_prediction(self, rates, data, percentile): flag = list() for ind, rate in enumerate(rates): if rate > 0.0: lower = poisson.ppf(percentile, rate) upper = poisson.ppf(1 - percentile, rate) if data[ind] < lower or data[ind] > upper: flag.append(1) else: flag.append(0) else: rospy.logwarn("Occurrence rate is %.2f" % rate) flag.append(-1) return flag def plot_region_idft(self, region): if self.spectrum_selection != 0: y, data = self.reconstruct_tof_from_spectrum( region, True, self.spectrum_selection) y2, _ = self.reconstruct_tof_from_spectrum(region, False, self.spectrum_selection) else: y, data = self.reconstruct_tof_from_spectrum(region, True) y2, _ = self.reconstruct_tof_from_spectrum(region, False) x = np.linspace(0, len(data), len(data)) xticks = time_ticks(self.minute_interval, self.window_interval, self.periodic_type) # y = map(lambda x: x / 100.0, y) # y2 = map(lambda x: x / 100.0, y2) # data = map(lambda x: x / 100.0, data) plt.plot(x, y, "-o", color="red", label="Amplitude Addition Model", linewidth=5) plt.plot(x, y2, "-x", color="blue", label="Best Amplitude Model", linewidth=5) plt.plot(x, data, "-*", color="green", label="Poisson Model", linewidth=5) plt.title("Reconstruction Occurrence Rate for Region %s" % region) # plt.xticks(x, xticks, rotation="horizontal", fontsize=30) plt.xticks(x, xticks, rotation="vertical") plt.xlabel( "One Week Period with %d minutes interval and %d window time" % (self.minute_interval, self.window_interval)) # plt.xticks([]) # plt.xticks(fontsize=30) # plt.yticks([]) # plt.yticks(fontsize=30) # plt.xlabel("Time", fontsize=40) plt.ylabel("Occurrence rate") # plt.ylabel("Amplitude", fontsize=40) plt.ylim(ymin=-5) # plt.legend(prop={'size': 40}, loc=4) plt.legend() plt.show()
class TrajectoryPeriodicity(object): def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10): """ Initialise analysis for trajectory periodicity. Initialise trajectory occurrence frequencies from database. """ self.tof = TrajectoryOccurrenceFrequencies( soma_map, soma_config, minute_interval, window_interval ) self.periodic_type = self.tof.periodic_type self.length_of_periodicity = len(self.tof.periodic_days) self.minute_interval = minute_interval self.window_interval = window_interval self.tof.load_tof() self.tof = self.tof.tof self.regions = self.tof.keys() self.spectrum_selection = 0 self.addition_technique = True def get_tof_values(self, region): """ Obtain trajectory occurrence frequency values for a specific region. """ length = (self.window_interval/self.minute_interval) - 1 y = list() region_tof = self.tof[region] for day, hourly_tof in region_tof.iteritems(): mins = sorted(hourly_tof) daily_y = list() for i in mins: daily_y.append(hourly_tof[i].get_occurrence_rate()) y.extend(daily_y[-length:] + daily_y[:-length]) return y def reconstruct_tof_from_spectrum(self, region, addition_method=True, num_of_freqs=30): """ Reconstruct trajectory occurrence frequency values after they are transformed into spectral dimensions. """ original = self.get_tof_values(region) if addition_method: spectrums, _ = self.get_significant_frequencies(original, num_of_freqs*2) spectrums = spectrums[0:num_of_freqs] else: spectrums = self.get_highest_n_freq(fft(original), num_of_freqs) reconstruction = 0 for spectrum in spectrums: reconstruction += self.rectify_wave( spectrum[2], spectrum[0], spectrum[1], len(original) ) reconstruction = map( lambda x: Lambda().get_occurrence_rate() if x <= 0.0 else x, reconstruction ) return reconstruction, original def model_selection(self, data, month, year, addition_method=True, max_freqs=30): spectrum_selection = list() validate_data = dict() for region in self.regions: validate_data.update({ region: trajectories_full_dates_periodic( data[region], month, year, self.length_of_periodicity, self.window_interval, self.minute_interval ) }) for num_of_freqs in range(1, max_freqs+1): mse_region = list() for region in self.regions: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, addition_method, num_of_freqs ) mse_recon, _ = self._calculate_mse( reconstruction_tof, original_tof, validate_data[region] ) mse_region.append(mse_recon) mse_region = [i for i in mse_region if i != -1] spectrum_selection.append(sum(mse_region) / float(len(mse_region))) spectrum_selection = spectrum_selection.index(min(spectrum_selection)) + 1 rospy.loginfo("Ideal total spectrums: %d" % spectrum_selection) self.spectrum_selection = spectrum_selection self.addition_technique = addition_method def calculate_mse(self, region, data, month, year): if self.spectrum_selection != 0: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, self.addition_technique, self.spectrum_selection ) else: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, self.addition_technique ) test_data = trajectories_full_dates_periodic( data, month, year, self.length_of_periodicity, self.window_interval, self.minute_interval ) reconstruction_tof = [i for j, i in enumerate(reconstruction_tof) if j % 5 == 0] original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0] test_data = [i for j, i in enumerate(test_data) if j % 5 == 0] # Dropping -1 in test_data together with corresponding tof test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data( test_data, reconstruction_tof, original_tof ) mse_recon = -1 mse_origin = -1 if len(reconstruction_tof) > 0 and len(original_tof) > 0: mse_recon = mse(test_data, reconstruction_tof) mse_origin = mse(test_data, original_tof) rospy.loginfo("Calculated MSE for original tof Region %s: %.2f" % (region, mse_origin)) rospy.loginfo("Calculated MSE for reconstruction tof Region %s: %.2f" % (region, mse_recon)) # temp_recon = np.sqrt(mse_recon) # sum_recon = 0 # for i in test_data: # sum_recon += (i - temp_recon)**2 # print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1))) # temp_recon = np.sqrt(mse_origin) # sum_recon = 0 # for i in test_data: # sum_recon += (i - temp_recon)**2 # print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1))) return mse_recon, mse_origin def rectify_wave(self, freq, amp, phs, num_of_points, up_thres=None, low_thres=None): xf = np.linspace(0.0, num_of_points, num_of_points) # frequency varations wave = amp * np.cos((freq * 2.0 * np.pi * xf) + phs) for ind, val in enumerate(wave): if low_thres is not None and val < low_thres: wave[ind] = low_thres if up_thres is not None and val > up_thres: wave[ind] = up_thres return wave def get_significant_frequencies(self, data, total_freq=15, max_addition=10, max_iteration=1000): N = len(data) xf = np.linspace(0.0, N, N) # initialise significant frequencies by taking frequency 0 spectrum_data = fft(data) [amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 1)[0] frequencies = [[amp, phs, freq]] freq_occur_counter = {freq: 1} exit_counter = 0 # data -= amp while len(frequencies) < total_freq: spectrum_data = fft(data) # recreate wave of the highest frequency [amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 2)[1] if freq == 0: [amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 2)[0] wave = amp * np.cos((freq * 2.0 * np.pi * xf) + phs) # substracting data with the wave data -= wave if freq not in zip(*frequencies)[2]: frequencies.append([amp, phs, freq]) freq_occur_counter.update({freq: 1}) else: for ind, val in enumerate(frequencies): if frequencies[ind][2] == freq and freq_occur_counter[freq] < max_addition: frequencies[ind][0] += amp frequencies[ind][1] = (( freq_occur_counter[freq] * frequencies[ind][1] ) + phs) / (freq_occur_counter[freq] + 1) freq_occur_counter[freq] += 1 exit_counter += 1 if exit_counter >= max_iteration: break return frequencies, data def get_highest_n_freq(self, freqs, n=15): N = len(freqs) freqs = freqs[0:N/2] indices = [i for i in range(len(freqs))] angles = np.angle(freqs) amplitudes = np.abs(freqs) / float(N) sorted_result = sorted(zip(amplitudes, angles, indices), reverse=True) n_freqs = sorted_result[:n] return n_freqs def _calculate_mse(self, reconstruction_tof, original_tof, test_data): # reconstruction_tof = [i for j, i in enumerate(reconstruction_tof) if j % 5 == 0] # original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0] # test_data = [i for j, i in enumerate(test_data) if j % 5 == 0] # Dropping -1 in test_data together with corresponding tof test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data( test_data, reconstruction_tof, original_tof ) mse_recon = -1 mse_origin = -1 if len(reconstruction_tof) > 0 and len(original_tof) > 0: mse_recon = mse(test_data, reconstruction_tof) mse_origin = mse(test_data, original_tof) return mse_recon, mse_origin def _remove_unobserved_data(self, data1, data2, data3): # Dropping -1 in data1, the corresponding indices in data2 and data3 are # also dropped deleted_indices = list() for ind, trajs in enumerate(data1): if trajs == -1: deleted_indices.append(ind) data1 = [j for i, j in enumerate(data1) if i not in deleted_indices] data2 = [j for i, j in enumerate(data2) if i not in deleted_indices] data3 = [j for i, j in enumerate(data3) if i not in deleted_indices] return data1, data2, data3, deleted_indices def prediction_accuracy(self, region, data, month, year, percentile=0.1, plot=False): if self.spectrum_selection != 0: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, self.addition_technique, self.spectrum_selection ) else: reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum( region, self.addition_technique # region, False, 26 ) test_data = trajectories_full_dates_periodic( data, month, year, self.length_of_periodicity, self.window_interval, self.minute_interval ) original_predict = self._get_prediction(original_tof, test_data, percentile) reconstr_predict = self._get_prediction(reconstruction_tof, test_data, percentile) _, clean_ori_pred, clean_recon_pred, indices = self._remove_unobserved_data( test_data, reconstr_predict, original_predict ) if len(clean_ori_pred) and len(clean_recon_pred): mse_predict = mse(clean_ori_pred, clean_recon_pred) else: mse_predict = -1 rospy.loginfo( "Calculated MSE for prediction between original and reconstruction for Region %s: %.2f" % (region, mse_predict) ) if plot: for index in indices: original_predict[index] = -1 reconstr_predict[index] = -1 x = np.linspace(0, len(test_data), len(test_data)) xticks = time_ticks(self.minute_interval, self.window_interval, self.periodic_type) plt.plot( x, original_predict, "-o", label="Prediction Original TOF" ) plt.plot( x, reconstr_predict, "-^", label="Prediction Reconstruction TOF" ) plt.title("Prediction for Region %s" % region) plt.xticks(x, xticks, rotation="vertical") plt.xlabel("One Week Period with %d minutes interval and %d window time" % (self.minute_interval, self.window_interval)) plt.ylabel("Prediction (1=Anomalous, 0=Normal, -1=Unobserved)") plt.ylim(ymin=-2, ymax=2) plt.legend() plt.show() def _get_prediction(self, rates, data, percentile): flag = list() for ind, rate in enumerate(rates): if rate > 0.0: lower = poisson.ppf(percentile, rate) upper = poisson.ppf(1-percentile, rate) if data[ind] < lower or data[ind] > upper: flag.append(1) else: flag.append(0) else: rospy.logwarn("Occurrence rate is %.2f" % rate) flag.append(-1) return flag def plot_region_idft(self, region): if self.spectrum_selection != 0: y, data = self.reconstruct_tof_from_spectrum( region, True, self.spectrum_selection ) y2, _ = self.reconstruct_tof_from_spectrum( region, False, self.spectrum_selection ) else: y, data = self.reconstruct_tof_from_spectrum(region, True) y2, _ = self.reconstruct_tof_from_spectrum(region, False) x = np.linspace(0, len(data), len(data)) xticks = time_ticks(self.minute_interval, self.window_interval, self.periodic_type) plt.plot( x, y, "-o", color="red", label="Amplitude Addition Reconstruction" ) plt.plot( x, y2, "-x", color="blue", label="Best Amplitude Reconstruction" ) plt.plot(x, data, "-", color="green", label="Original Model") plt.title("Reconstruction Occurrence Rate for Region %s" % region) plt.xticks(x, xticks, rotation="vertical") plt.xlabel("One Week Period with %d minutes interval and %d window time" % (self.minute_interval, self.window_interval)) plt.ylabel("Occurrence rate value") plt.ylim(ymin=-5) plt.legend() plt.show()
class TOFPlot(object): """ This class is for plotting the trajectory occurrence frequency every n minute with m window minute interval (as its bin) that is continuous starting from Monday 00:00 to Sunday 23:59 """ def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10): """ Initialize plotting, it needs the name of the soma map, specific soma configuration, the minute interval between two different occurrence rate, and the window interval that acts as bins. """ self.tof = TrajectoryOccurrenceFrequencies(soma_map, soma_config, minute_interval, window_interval) self.xticks = time_ticks(minute_interval, window_interval, self.tof.periodic_type) self.x = np.arange(len(self.xticks)) self.minute_interval = minute_interval self.window_interval = window_interval self.periodic_length = len(self.tof.periodic_days) self.tof.load_tof() self.tof = self.tof.tof self.regions = self.tof.keys() self.colors = [ (0., 0., 0.), (0., 0., 1.), (0., 1., 0.), (0., 1., 1.), (1., 0., 0.), (1., 0., 1.), (1., 1., 0.), (.75, .75, .75), (0., 0., 0.), (0., 0., .5), (0., .5, 0.), (0., .5, .5), (.5, 0., 0.), (.5, 0., .5), (.5, .5, 0.), (.25, .25, .25) ] def get_y_yerr_per_region(self, region): """ Obtain the occurrence rate value, the mode of gamma distribution, for each region including the lower percentile 0.025, and the upper percentile of the value 0.975 showing the wide of the gamma distribution. """ length = (self.window_interval/self.minute_interval) - 1 y = list() lower_percentile = list() upper_percentile = list() region_tof = self.tof[region] for day, hourly_tof in region_tof.iteritems(): mins = sorted(hourly_tof) daily_y = list() daily_low = list() daily_up = list() for i in mins: daily_y.append(hourly_tof[i].get_occurrence_rate()) daily_low.append( abs(hourly_tof[i].get_occurrence_rate() - gamma.ppf(0.025, hourly_tof[i].occurrence_shape, scale=1/float(hourly_tof[i].occurrence_scale))) # gamma.ppf(0.025, mins_tof[i].occurrence_shape, scale=1/float(mins_tof[i].occurrence_scale)) ) daily_up.append( abs(hourly_tof[i].get_occurrence_rate() - gamma.ppf(0.975, hourly_tof[i].occurrence_shape, scale=1/float(hourly_tof[i].occurrence_scale))) # gamma.ppf(0.975, mins_tof[i].occurrence_shape, scale=1/float(mins_tof[i].occurrence_scale)) ) y.extend(daily_y[-length:] + daily_y[:-length]) lower_percentile.extend(daily_low[-length:] + daily_low[:-length]) upper_percentile.extend(daily_up[-length:] + daily_up[:-length]) return y, lower_percentile, upper_percentile def show_tof_per_region(self, region): """ Show the occurrence rate over a week/month for each region available from soma map """ y, low_err, up_err = self.get_y_yerr_per_region(region) plt.errorbar( self.x, y, yerr=[low_err, up_err], color='b', ecolor='r', fmt="-o", label="Region " + region # fmt="-o", label="Poisson Model" ) # plt.title("Poisson Processes of the Corridor", fontsize=40) plt.title("Occurrence Rate for Region %s" % region) # plt.xticks(self.x, self.xticks, rotation="horizontal", fontsize=40) plt.xticks(self.x, self.xticks, rotation="vertical") plt.xlabel( "One Week Period with %d minutes interval and %d window time" % ( self.minute_interval, self.window_interval ) ) # plt.ylabel("Arrival Rate", fontsize=40) plt.ylabel("Occurrence rate value") plt.ylim(ymin=-1) # plt.legend(prop={'size': 40}) plt.legend() plt.show() def show_tof(self): """ Show occurrence rate for all regions over a week/month """ for region in self.regions: try: color = self.colors[int(region) % len(self.colors)] ecolor = self.colors[(int(region)**2 + 4) % len(self.colors)] except: color = (0., 0., 1.) ecolor = (1., 0., 0.) y, low_err, up_err = self.get_y_yerr_per_region(region) plt.errorbar( self.x, y, yerr=[low_err, up_err], color=color, ecolor=ecolor, fmt="-o", label="Region " + region ) plt.title("Occurrence Rate for All Regions") plt.xticks(self.x, self.xticks, rotation="vertical") plt.xlabel( "One Week Period with %d minutes interval and %d window time" % ( self.minute_interval, self.window_interval ) ) plt.ylabel("Occurrence rate value") plt.ylim(ymin=-1) plt.legend() plt.show()