def __init__(self,
name,
soma_map,
soma_config,
minute_interval,
window_interval,
collection="occurrence_rates"):
self.topo_map = None
self.soma_map = soma_map
self.soma_config = soma_config
self.minute_interval = minute_interval
self.window_interval = window_interval
self.rotm = RegionObservationTimeManager(soma_map, soma_config)
self.tre = TrajectoryRegionEstimate(soma_map, soma_config,
minute_interval)
self.tof = TrajectoryOccurrenceFrequencies(
soma_map,
soma_config,
minute_interval=minute_interval,
window_interval=window_interval)
self.gs = GeoSpatialStoreProxy('geospatial_store', 'soma')
rospy.loginfo("Connect to database collection %s..." % collection)
self._ms = MessageStoreProxy(collection=collection)
self._soma_db = MessageStoreProxy(collection="soma_roi")
rospy.loginfo("Create a service %s/service..." % name)
self.service = rospy.Service(name + '/service',
TrajectoryOccurrenceRate, self.srv_cb)
rospy.loginfo("Create an action server %s..." % name)
self._as = actionlib.SimpleActionServer(name,
OccurrenceRateLearningAction,
execute_cb=self.execute,
auto_start=False)
self._as.start()
def __init__(self,
soma_map,
soma_config,
minute_interval=1,
window_interval=10):
"""
Initialise analysis for trajectory periodicity.
Initialise trajectory occurrence frequencies from database.
"""
self.tof = TrajectoryOccurrenceFrequencies(soma_map, soma_config,
minute_interval,
window_interval)
self.periodic_type = self.tof.periodic_type
self.length_of_periodicity = len(self.tof.periodic_days)
self.minute_interval = minute_interval
self.window_interval = window_interval
self.tof.load_tof()
self.tof = self.tof.tof
self.regions = self.tof.keys()
self.spectrum_selection = 0
self.addition_technique = True
def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10):
"""
Initialize plotting, it needs the name of the soma map, specific soma configuration,
the minute interval between two different occurrence rate, and the window interval that
acts as bins.
"""
self.tof = TrajectoryOccurrenceFrequencies(soma_map, soma_config, minute_interval, window_interval)
self.xticks = time_ticks(minute_interval, window_interval, self.tof.periodic_type)
self.x = np.arange(len(self.xticks))
self.minute_interval = minute_interval
self.window_interval = window_interval
self.periodic_length = len(self.tof.periodic_days)
self.tof.load_tof()
self.tof = self.tof.tof
self.regions = self.tof.keys()
self.colors = [
(0., 0., 0.), (0., 0., 1.), (0., 1., 0.), (0., 1., 1.),
(1., 0., 0.), (1., 0., 1.), (1., 1., 0.), (.75, .75, .75),
(0., 0., 0.), (0., 0., .5), (0., .5, 0.), (0., .5, .5),
(.5, 0., 0.), (.5, 0., .5), (.5, .5, 0.), (.25, .25, .25)
]
def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10):
"""
Initialise analysis for trajectory periodicity.
Initialise trajectory occurrence frequencies from database.
"""
self.tof = TrajectoryOccurrenceFrequencies(
soma_map, soma_config, minute_interval, window_interval
)
self.periodic_type = self.tof.periodic_type
self.length_of_periodicity = len(self.tof.periodic_days)
self.minute_interval = minute_interval
self.window_interval = window_interval
self.tof.load_tof()
self.tof = self.tof.tof
self.regions = self.tof.keys()
self.spectrum_selection = 0
self.addition_technique = True
def __init__(self, name, soma_map, soma_config, minute_interval, window_interval, collection="occurrence_rates"):
self.soma_map = soma_map
self.soma_config = soma_config
self.minute_interval = minute_interval
self.window_interval = window_interval
self.rotm = RegionObservationTimeManager(soma_map, soma_config)
self.tre = TrajectoryRegionEstimate(soma_map, soma_config, minute_interval)
self.tof = TrajectoryOccurrenceFrequencies(
soma_map, soma_config, minute_interval=minute_interval, window_interval=window_interval
)
rospy.loginfo("Connect to database collection %s..." % collection)
self._ms = MessageStoreProxy(collection=collection)
rospy.loginfo("Create a service %s/service..." % name)
self.service = rospy.Service(name + "/service", TrajectoryOccurrenceRate, self.srv_cb)
rospy.loginfo("Create an action server %s..." % name)
self._as = actionlib.SimpleActionServer(
name, OccurrenceRateLearningAction, execute_cb=self.execute, auto_start=False
)
self._as.start()
if __name__ == '__main__':
rospy.init_node("synthetic_tof")
if len(sys.argv) < 4:
rospy.logerr("usage: synthetic minute_interval window_interval store(0)/test(1)")
sys.exit(2)
interval = int(sys.argv[1])
window = int(sys.argv[2])
sw = SyntheticWave(interval)
if not int(sys.argv[3]):
waves = sw.get_one_month_synthetic_wave(True)
tof = TOF("synthetic", "synthetic_config", interval, window)
tof.load_tof()
for i in range(4, 31+1):
prev_traj_est = trajectory_estimate_for_date(
waves, datetime.date(2015, 5, i-1)
)
curr_traj_est = trajectory_estimate_for_date(
waves, datetime.date(2015, 5, i)
)
tof.update_tof_daily(
curr_traj_est, prev_traj_est, datetime.date(2015, 5, i)
)
tof.store_tof()
else:
waves = sw.get_one_week_synthetic_wave(True)
tp = TrajectoryPeriodicity("synthetic", "synthetic_config", interval, window)
class TrajectoryArrivalRateManager(object):
def __init__(self,
name,
soma_map,
soma_config,
minute_interval,
window_interval,
collection="occurrence_rates"):
self.topo_map = None
self.soma_map = soma_map
self.soma_config = soma_config
self.minute_interval = minute_interval
self.window_interval = window_interval
self.rotm = RegionObservationTimeManager(soma_map, soma_config)
self.tre = TrajectoryRegionEstimate(soma_map, soma_config,
minute_interval)
self.tof = TrajectoryOccurrenceFrequencies(
soma_map,
soma_config,
minute_interval=minute_interval,
window_interval=window_interval)
self.gs = GeoSpatialStoreProxy('geospatial_store', 'soma')
rospy.loginfo("Connect to database collection %s..." % collection)
self._ms = MessageStoreProxy(collection=collection)
self._soma_db = MessageStoreProxy(collection="soma_roi")
rospy.loginfo("Create a service %s/service..." % name)
self.service = rospy.Service(name + '/service',
TrajectoryOccurrenceRate, self.srv_cb)
rospy.loginfo("Create an action server %s..." % name)
self._as = actionlib.SimpleActionServer(name,
OccurrenceRateLearningAction,
execute_cb=self.execute,
auto_start=False)
self._as.start()
def execute(self, goal):
temp_start_time = rospy.Time.now()
curr_date = datetime.datetime.fromtimestamp(rospy.Time.now().secs)
# curr_date = datetime.datetime(2016, 3, 10, 0, 50)
if curr_date.hour >= 0 and curr_date.hour < 8:
curr_date = curr_date - datetime.timedelta(hours=24)
curr_date = datetime.datetime(curr_date.year, curr_date.month,
curr_date.day, 0, 0)
prev_date = curr_date - datetime.timedelta(hours=24)
self.rotm.calculate_region_observation_duration([prev_date, curr_date],
self.minute_interval)
self.rotm.store_to_mongo()
if self._as.is_preempt_requested():
self._as.set_preempted()
return
curr_traj_est = self.tre.estimate_trajectories_daily([curr_date.day],
curr_date.month,
curr_date.year)
curr_traj_est = trajectory_estimate_for_date(curr_traj_est, curr_date)
prev_traj_est = self.tre.estimate_trajectories_daily([prev_date.day],
prev_date.month,
prev_date.year)
prev_traj_est = trajectory_estimate_for_date(prev_traj_est, prev_date)
if self._as.is_preempt_requested():
self._as.set_preempted()
return
self.tof.update_tof_daily(curr_traj_est, prev_traj_est, curr_date)
self.tof.store_tof()
self.rotm.reinit()
self.tof.reinit()
temp_end_time = rospy.Time.now()
rospy.loginfo("Time needed to complete this is %d" %
(temp_end_time - temp_start_time).secs)
self._as.set_succeeded(OccurrenceRateLearningResult())
def _get_occurrence_rate_logs(self, start_time, end_time):
filtered_logs = list()
if (end_time -
start_time).total_seconds() <= self.window_interval * 60:
end_time = start_time + datetime.timedelta(
seconds=self.minute_interval * 60)
if start_time.hour == end_time.hour and start_time.day == end_time.day:
query = {
"soma": self.soma_map,
"soma_config": self.soma_config,
"duration.secs": self.window_interval * 60,
"day": start_time.weekday(),
"hour": start_time.hour,
"minute": {
"$gte": start_time.minute,
"$lt": end_time.minute
}
}
logs = self._ms.query(OccurrenceRate._type, message_query=query)
filtered_logs = logs
elif start_time.hour != end_time.hour and start_time.day == end_time.day:
query = {
"soma": self.soma_map,
"soma_config": self.soma_config,
"duration.secs": self.window_interval * 60,
"day": start_time.weekday(),
"hour": {
"$gte": start_time.hour,
"$lte": end_time.hour
},
}
logs = self._ms.query(OccurrenceRate._type, message_query=query)
for log in logs:
if log[0].hour == start_time.hour and log[
0].minute >= start_time.minute:
filtered_logs.append(log)
elif log[0].hour == end_time.hour and log[
0].minute < end_time.minute:
filtered_logs.append(log)
elif log[0].hour > start_time.hour and log[
0].hour < end_time.hour:
filtered_logs.append(log)
else:
day = start_time.weekday()
end_day = end_time.weekday()
query = {
"soma": self.soma_map,
"soma_config": self.soma_config,
"duration.secs": self.window_interval * 60,
"day": {
"$gte": day,
"$lte": end_day
}
}
logs = self._ms.query(OccurrenceRate._type, message_query=query)
for log in logs:
if log[0].day == day and log[0].hour >= start_time.hour and log[
0].minute >= start_time.minute:
filtered_logs.append(log)
if log[0].day == end_day and log[
0].hour <= end_time.hour and log[
0].minute < end_time.minute:
filtered_logs.append(log)
elif end_day > day:
if log[0].day > day and log[0].day < end_day:
filtered_logs.append(log)
elif end_day < day:
if log[0].day > day:
filtered_logs.append(log)
elif log[0].day < end_day:
filtered_logs.append(log)
return filtered_logs
def _choose_proper_region_to_observe(self, rois, wp_point):
accumulate_dist = dict()
for roi in rois:
query = {
"map": self.soma_map,
"config": self.soma_config,
"roi_id": roi
}
logs = self._soma_db.query(SOMAROIObject._type,
message_query=query)
for log in logs:
if roi not in accumulate_dist:
accumulate_dist[roi] = 0.0
accumulate_dist[roi] = distance_calc.euclidean(
[wp_point.pose.position.x, wp_point.pose.position.y],
[log[0].pose.position.x, log[0].pose.position.y])
accumulate_dist = sorted(accumulate_dist,
key=lambda i: accumulate_dist[i],
reverse=True)
return accumulate_dist[-1]
def _get_waypoints(self, region_ids):
waypoints = dict()
topo_sub = rospy.Subscriber("/topological_map", TopologicalMap,
self._topo_map_cb, None, 10)
while self.topo_map is None:
rospy.sleep(0.1)
rospy.logwarn("Trying to get information from /topological_map...")
topo_sub.unregister()
for wp in self.topo_map.nodes:
if wp.name == "ChargingPoint":
continue
_, _, yaw = euler_from_quaternion(
[0, 0, wp.pose.orientation.z, wp.pose.orientation.w])
coords = robot_view_cone(wp.pose.position.x, wp.pose.position.y,
yaw)
langitude_latitude = list()
for pt in coords:
langitude_latitude.append(
self.gs.coords_to_lnglat(pt[0], pt[1]))
langitude_latitude.append(
self.gs.coords_to_lnglat(coords[0][0], coords[0][1]))
if self.gs.observed_roi(langitude_latitude, self.soma_map,
self.soma_config).count() > 1:
rospy.logwarn(
"There are two or more regions covered by the sight of the robot in %s"
% wp.name)
roi = list()
for j in self.gs.observed_roi(langitude_latitude, self.soma_map,
self.soma_config):
roi.append(str(j['soma_roi_id']))
roi = [i for i in roi if i in region_ids]
if len(roi) < 1:
continue
elif len(roi) == 1:
waypoints[roi[0]] = wp.name
else:
roi = self._choose_proper_region_to_observe(roi, wp)
waypoints[roi] = wp.name
return waypoints
def _topo_map_cb(self, topo_map):
self.topo_map = topo_map
def _get_long_duration(self, four_tuple, start_time):
datetimes = list()
for i in four_tuple:
delta_day = (i[0] - start_time.weekday()) % 7
curr_date = start_time + datetime.timedelta(days=delta_day)
curr_date = datetime.datetime(curr_date.year, curr_date.month,
curr_date.day, i[1], i[2])
datetimes.append(curr_date)
datetimes = sorted(datetimes)
counter = 0
for i in range(1, len(datetimes)):
if (datetimes[i] -
datetimes[i - 1]).seconds == self.minute_interval * 60:
counter += 1
elif (datetimes[i] -
datetimes[i - 1]).seconds <= self.window_interval * 60:
counter += (datetimes[i] - datetimes[i - 1]).seconds / (
self.minute_interval * 60)
duration = rospy.Duration(self.window_interval * 60 +
(counter * self.minute_interval * 60))
best_time = rospy.Time(time.mktime(datetimes[0].timetuple()))
return best_time, duration
def _get_best_regions(self, logs, start_time):
or_regions = dict()
best_time = dict()
for log in logs:
if log[0].region_id not in or_regions:
or_regions[log[0].region_id] = 0.0
or_regions[log[0].region_id] += log[0].occurrence_rate
if log[0].region_id not in best_time:
best_time[log[0].region_id] = [[
log[0].day, log[0].hour, log[0].minute,
log[0].occurrence_rate
]]
elif best_time[log[0].region_id][-1][-1] < log[0].occurrence_rate:
best_time[log[0].region_id][-1] = [
log[0].day, log[0].hour, log[0].minute,
log[0].occurrence_rate
]
elif best_time[log[0].region_id][-1][-1] == log[0].occurrence_rate:
best_time[log[0].region_id].append([
log[0].day, log[0].hour, log[0].minute,
log[0].occurrence_rate
])
duration = dict()
for roi in best_time.keys():
temp = best_time[roi]
if len(temp) > 0:
best_time[roi], duration[roi] = self._get_long_duration(
temp, start_time)
else:
rospy.logwarn(
"There is no recommended visiting time for region %s" %
roi)
duration[roi] = rospy.Duration(0)
best_time[roi] = rospy.Time.now()
return or_regions, best_time, duration
def srv_cb(self, msg):
region_ids = list()
best_times_list = list()
durations_list = list()
waypoints_list = list()
rospy.loginfo("Got a request...")
print msg
rospy.loginfo(
"Retrieving trajectory occurrence frequencies from database...")
start_time = datetime.datetime.fromtimestamp(msg.start_time.secs)
end_time = datetime.datetime.fromtimestamp(msg.end_time.secs)
if end_time >= start_time:
logs = self._get_occurrence_rate_logs(start_time, end_time)
# calculate occurrence_rate per region
or_regions, best_times, durations = self._get_best_regions(
logs, start_time)
region_ids = sorted(or_regions,
key=lambda i: or_regions[i],
reverse=True)
# get the nearest waypoints somehow
waypoints = self._get_waypoints(region_ids)
# ordering
for roi in region_ids:
best_times_list.append(best_times[roi])
durations_list.append(durations[roi])
waypoints_list.append(waypoints[roi])
rospy.loginfo("Composing answer...")
print TrajectoryOccurrenceRateResponse(
region_ids[:msg.n_best], best_times_list[:msg.n_best],
durations_list[:msg.n_best], waypoints_list[:msg.n_best])
return TrajectoryOccurrenceRateResponse(region_ids[:msg.n_best],
best_times_list[:msg.n_best],
durations_list[:msg.n_best],
waypoints_list[:msg.n_best])
rospy.init_node("synthetic_tof")
if len(sys.argv) < 4:
rospy.logerr(
"usage: synthetic minute_interval window_interval store(0)/test(1)"
)
sys.exit(2)
interval = int(sys.argv[1])
window = int(sys.argv[2])
sw = SyntheticWave(interval)
if not int(sys.argv[3]):
waves = sw.get_one_month_synthetic_wave(True)
# waves = sw.get_one_month_synthetic_wave(False) # no noise
tof = TOF("synthetic", "synthetic_config", interval, window)
tof.load_tof()
for i in range(4, 31 + 1):
prev_traj_est = trajectory_estimate_for_date(
waves, datetime.date(2015, 5, i - 1))
curr_traj_est = trajectory_estimate_for_date(
waves, datetime.date(2015, 5, i))
tof.update_tof_daily(curr_traj_est, prev_traj_est,
datetime.date(2015, 5, i))
tof.store_tof()
else:
waves = sw.get_one_week_synthetic_wave(True)
tp = TrajectoryPeriodicity("synthetic", "synthetic_config", interval,
window)
# comment from here if just want the plot
inp = raw_input("MSE(0) or Prediction MSE(1): ")
class TrajectoryOccurrenceRateManager(object):
def __init__(self, name, soma_map, soma_config, minute_interval, window_interval, collection="occurrence_rates"):
self.soma_map = soma_map
self.soma_config = soma_config
self.minute_interval = minute_interval
self.window_interval = window_interval
self.rotm = RegionObservationTimeManager(soma_map, soma_config)
self.tre = TrajectoryRegionEstimate(soma_map, soma_config, minute_interval)
self.tof = TrajectoryOccurrenceFrequencies(
soma_map, soma_config, minute_interval=minute_interval, window_interval=window_interval
)
rospy.loginfo("Connect to database collection %s..." % collection)
self._ms = MessageStoreProxy(collection=collection)
rospy.loginfo("Create a service %s/service..." % name)
self.service = rospy.Service(name + "/service", TrajectoryOccurrenceRate, self.srv_cb)
rospy.loginfo("Create an action server %s..." % name)
self._as = actionlib.SimpleActionServer(
name, OccurrenceRateLearningAction, execute_cb=self.execute, auto_start=False
)
self._as.start()
def execute(self, goal):
temp_start_time = rospy.Time.now()
curr_date = datetime.datetime.fromtimestamp(rospy.Time.now().secs)
# curr_date = datetime.datetime(2016, 3, 10, 0, 50)
if curr_date.hour >= 0 and curr_date.hour < 8:
curr_date = curr_date - datetime.timedelta(hours=24)
curr_date = datetime.datetime(curr_date.year, curr_date.month, curr_date.day, 0, 0)
prev_date = curr_date - datetime.timedelta(hours=24)
self.rotm.calculate_region_observation_duration([prev_date, curr_date], self.minute_interval)
self.rotm.store_to_mongo()
if self._as.is_preempt_requested():
self._as.set_preempted()
return
curr_traj_est = self.tre.estimate_trajectories_daily([curr_date.day], curr_date.month, curr_date.year)
curr_traj_est = trajectory_estimate_for_date(curr_traj_est, curr_date)
prev_traj_est = self.tre.estimate_trajectories_daily([prev_date.day], prev_date.month, prev_date.year)
prev_traj_est = trajectory_estimate_for_date(prev_traj_est, prev_date)
if self._as.is_preempt_requested():
self._as.set_preempted()
return
self.tof.update_tof_daily(curr_traj_est, prev_traj_est, curr_date)
self.tof.store_tof()
self.rotm.reinit()
self.tof.reinit()
temp_end_time = rospy.Time.now()
rospy.loginfo("Time needed to complete this is %d" % (temp_end_time - temp_start_time).secs)
self._as.set_succeeded(OccurrenceRateLearningResult())
def srv_cb(self, msg):
rospy.loginfo("Retrieving trajectory occurrence frequencies from database...")
print msg
query = {
"soma": self.soma_map,
"soma_config": self.soma_config,
"region_id": msg.region_id,
"day": msg.day,
"end_hour": msg.hour,
"end_minute": msg.minute,
}
logs = self._ms.query(OccurrenceRate._type, message_query=query)
occurrence_rate = 0.0
counter = 0
for log in logs:
occurrence_rate += log[0].occurrence_rate
counter += 1
if counter == 0:
counter += 1
if len(logs) > 0:
rospy.loginfo("Occurrence Rate is obtained, which has value %.2f" % (occurrence_rate / float(counter)))
else:
rospy.loginfo("No occurrence rate is obtained, returning zero value")
return TrajectoryOccurrenceRateResponse(occurrence_rate / float(counter))
class TrajectoryPeriodicity(object):
def __init__(self,
soma_map,
soma_config,
minute_interval=1,
window_interval=10):
"""
Initialise analysis for trajectory periodicity.
Initialise trajectory occurrence frequencies from database.
"""
self.tof = TrajectoryOccurrenceFrequencies(soma_map, soma_config,
minute_interval,
window_interval)
self.periodic_type = self.tof.periodic_type
self.length_of_periodicity = len(self.tof.periodic_days)
self.minute_interval = minute_interval
self.window_interval = window_interval
self.tof.load_tof()
self.tof = self.tof.tof
self.regions = self.tof.keys()
self.spectrum_selection = 0
self.addition_technique = True
def get_tof_values(self, region):
"""
Obtain trajectory occurrence frequency values for a specific region.
"""
length = (self.window_interval / self.minute_interval) - 1
y = list()
region_tof = self.tof[region]
for day, hourly_tof in region_tof.iteritems():
mins = sorted(hourly_tof)
daily_y = list()
for i in mins:
daily_y.append(hourly_tof[i].get_occurrence_rate())
y.extend(daily_y[-length:] + daily_y[:-length])
return y
def reconstruct_tof_from_spectrum(self,
region,
addition_method=True,
num_of_freqs=30):
"""
Reconstruct trajectory occurrence frequency values after they are transformed
into spectral dimensions.
"""
original = self.get_tof_values(region)
if addition_method:
spectrums, _ = self.get_significant_frequencies(
original, num_of_freqs * 2)
spectrums = spectrums[0:num_of_freqs]
else:
spectrums = self.get_highest_n_freq(fft(original), num_of_freqs)
reconstruction = 0
for spectrum in spectrums:
reconstruction += self.rectify_wave(spectrum[2], spectrum[0],
spectrum[1], len(original))
reconstruction = map(
lambda x: Lambda().get_occurrence_rate()
if x <= 0.0 else x, reconstruction)
return reconstruction, original
def model_selection(self,
data,
month,
year,
addition_method=True,
max_freqs=30):
spectrum_selection = list()
validate_data = dict()
for region in self.regions:
validate_data.update({
region:
trajectories_full_dates_periodic(data[region], month, year,
self.length_of_periodicity,
self.window_interval,
self.minute_interval)
})
for num_of_freqs in range(1, max_freqs + 1):
mse_region = list()
for region in self.regions:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, addition_method, num_of_freqs)
mse_recon, _ = self._calculate_mse(reconstruction_tof,
original_tof,
validate_data[region])
mse_region.append(mse_recon)
mse_region = [i for i in mse_region if i != -1]
spectrum_selection.append(sum(mse_region) / float(len(mse_region)))
spectrum_selection = spectrum_selection.index(
min(spectrum_selection)) + 1
rospy.loginfo("Ideal total spectrums: %d" % spectrum_selection)
self.spectrum_selection = spectrum_selection
self.addition_technique = addition_method
def calculate_mse(self, region, data, month, year):
if self.spectrum_selection != 0:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique, self.spectrum_selection)
else:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique)
test_data = trajectories_full_dates_periodic(
data, month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval)
reconstruction_tof = [
i for j, i in enumerate(reconstruction_tof) if j % 5 == 0
]
original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0]
test_data = [i for j, i in enumerate(test_data) if j % 5 == 0]
# Dropping -1 in test_data together with corresponding tof
test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data(
test_data, reconstruction_tof, original_tof)
mse_recon = -1
mse_origin = -1
if len(reconstruction_tof) > 0 and len(original_tof) > 0:
mse_recon = mse(test_data, reconstruction_tof)
mse_origin = mse(test_data, original_tof)
rospy.loginfo("Calculated MSE for original tof Region %s: %.2f" %
(region, mse_origin))
rospy.loginfo("Calculated MSE for reconstruction tof Region %s: %.2f" %
(region, mse_recon))
# temp_recon = np.sqrt(mse_recon)
# sum_recon = 0
# for i in test_data:
# sum_recon += (i - temp_recon)**2
# print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1)))
# temp_recon = np.sqrt(mse_origin)
# sum_recon = 0
# for i in test_data:
# sum_recon += (i - temp_recon)**2
# print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1)))
return mse_recon, mse_origin
def rectify_wave(self,
freq,
amp,
phs,
num_of_points,
up_thres=None,
low_thres=None):
xf = np.linspace(0.0, num_of_points,
num_of_points) # frequency varations
wave = amp * np.cos((freq * 2.0 * np.pi * xf) + phs)
for ind, val in enumerate(wave):
if low_thres is not None and val < low_thres:
wave[ind] = low_thres
if up_thres is not None and val > up_thres:
wave[ind] = up_thres
return wave
def get_significant_frequencies(self,
data,
total_freq=15,
max_addition=10,
max_iteration=1000):
N = len(data)
xf = np.linspace(0.0, N, N)
# initialise significant frequencies by taking frequency 0
spectrum_data = fft(data)
[amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 1)[0]
frequencies = [[amp, phs, freq]]
freq_occur_counter = {freq: 1}
exit_counter = 0
# data -= amp
while len(frequencies) < total_freq:
spectrum_data = fft(data)
# recreate wave of the highest frequency
[amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 2)[1]
if freq == 0:
[amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 2)[0]
wave = amp * np.cos((freq * 2.0 * np.pi * xf) + phs)
# substracting data with the wave
data -= wave
if freq not in zip(*frequencies)[2]:
frequencies.append([amp, phs, freq])
freq_occur_counter.update({freq: 1})
else:
for ind, val in enumerate(frequencies):
if frequencies[ind][2] == freq and freq_occur_counter[
freq] < max_addition:
frequencies[ind][0] += amp
frequencies[ind][1] = (
(freq_occur_counter[freq] * frequencies[ind][1]) +
phs) / (freq_occur_counter[freq] + 1)
freq_occur_counter[freq] += 1
exit_counter += 1
if exit_counter >= max_iteration:
break
return frequencies, data
def get_highest_n_freq(self, freqs, n=15):
N = len(freqs)
freqs = freqs[0:N / 2]
indices = [i for i in range(len(freqs))]
angles = np.angle(freqs)
amplitudes = np.abs(freqs) / float(N)
sorted_result = sorted(zip(amplitudes, angles, indices), reverse=True)
n_freqs = sorted_result[:n]
return n_freqs
def _calculate_mse(self, reconstruction_tof, original_tof, test_data):
# reconstruction_tof = [i for j, i in enumerate(reconstruction_tof) if j % 5 == 0]
# original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0]
# test_data = [i for j, i in enumerate(test_data) if j % 5 == 0]
# Dropping -1 in test_data together with corresponding tof
test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data(
test_data, reconstruction_tof, original_tof)
mse_recon = -1
mse_origin = -1
if len(reconstruction_tof) > 0 and len(original_tof) > 0:
mse_recon = mse(test_data, reconstruction_tof)
mse_origin = mse(test_data, original_tof)
return mse_recon, mse_origin
def _remove_unobserved_data(self, data1, data2, data3):
# Dropping -1 in data1, the corresponding indices in data2 and data3 are
# also dropped
deleted_indices = list()
for ind, trajs in enumerate(data1):
if trajs == -1:
deleted_indices.append(ind)
data1 = [j for i, j in enumerate(data1) if i not in deleted_indices]
data2 = [j for i, j in enumerate(data2) if i not in deleted_indices]
data3 = [j for i, j in enumerate(data3) if i not in deleted_indices]
return data1, data2, data3, deleted_indices
def prediction_accuracy(self,
region,
data,
month,
year,
percentile=0.1,
plot=False):
if self.spectrum_selection != 0:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique, self.spectrum_selection)
else:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique
# region, False, 26
)
test_data = trajectories_full_dates_periodic(
data, month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval)
original_predict = self._get_prediction(original_tof, test_data,
percentile)
reconstr_predict = self._get_prediction(reconstruction_tof, test_data,
percentile)
_, clean_ori_pred, clean_recon_pred, indices = self._remove_unobserved_data(
test_data, reconstr_predict, original_predict)
if len(clean_ori_pred) and len(clean_recon_pred):
mse_predict = mse(clean_ori_pred, clean_recon_pred)
else:
mse_predict = -1
rospy.loginfo(
"Calculated MSE for prediction between original and reconstruction for Region %s: %.2f"
% (region, mse_predict))
if plot:
for index in indices:
original_predict[index] = -1
reconstr_predict[index] = -1
x = np.linspace(0, len(test_data), len(test_data))
xticks = time_ticks(self.minute_interval, self.window_interval,
self.periodic_type)
plt.plot(x,
original_predict,
"-o",
label="Prediction Original TOF")
plt.plot(x,
reconstr_predict,
"-^",
label="Prediction Reconstruction TOF")
plt.title("Prediction for Region %s" % region)
plt.xticks(x, xticks, rotation="vertical")
plt.xlabel(
"One Week Period with %d minutes interval and %d window time" %
(self.minute_interval, self.window_interval))
plt.ylabel("Prediction (1=Anomalous, 0=Normal, -1=Unobserved)")
plt.ylim(ymin=-2, ymax=2)
plt.legend()
plt.show()
def _get_prediction(self, rates, data, percentile):
flag = list()
for ind, rate in enumerate(rates):
if rate > 0.0:
lower = poisson.ppf(percentile, rate)
upper = poisson.ppf(1 - percentile, rate)
if data[ind] < lower or data[ind] > upper:
flag.append(1)
else:
flag.append(0)
else:
rospy.logwarn("Occurrence rate is %.2f" % rate)
flag.append(-1)
return flag
def plot_region_idft(self, region):
if self.spectrum_selection != 0:
y, data = self.reconstruct_tof_from_spectrum(
region, True, self.spectrum_selection)
y2, _ = self.reconstruct_tof_from_spectrum(region, False,
self.spectrum_selection)
else:
y, data = self.reconstruct_tof_from_spectrum(region, True)
y2, _ = self.reconstruct_tof_from_spectrum(region, False)
x = np.linspace(0, len(data), len(data))
xticks = time_ticks(self.minute_interval, self.window_interval,
self.periodic_type)
# y = map(lambda x: x / 100.0, y)
# y2 = map(lambda x: x / 100.0, y2)
# data = map(lambda x: x / 100.0, data)
plt.plot(x,
y,
"-o",
color="red",
label="Amplitude Addition Model",
linewidth=5)
plt.plot(x,
y2,
"-x",
color="blue",
label="Best Amplitude Model",
linewidth=5)
plt.plot(x,
data,
"-*",
color="green",
label="Poisson Model",
linewidth=5)
plt.title("Reconstruction Occurrence Rate for Region %s" % region)
# plt.xticks(x, xticks, rotation="horizontal", fontsize=30)
plt.xticks(x, xticks, rotation="vertical")
plt.xlabel(
"One Week Period with %d minutes interval and %d window time" %
(self.minute_interval, self.window_interval))
# plt.xticks([])
# plt.xticks(fontsize=30)
# plt.yticks([])
# plt.yticks(fontsize=30)
# plt.xlabel("Time", fontsize=40)
plt.ylabel("Occurrence rate")
# plt.ylabel("Amplitude", fontsize=40)
plt.ylim(ymin=-5)
# plt.legend(prop={'size': 40}, loc=4)
plt.legend()
plt.show()
class TrajectoryPeriodicity(object):
def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10):
"""
Initialise analysis for trajectory periodicity.
Initialise trajectory occurrence frequencies from database.
"""
self.tof = TrajectoryOccurrenceFrequencies(
soma_map, soma_config, minute_interval, window_interval
)
self.periodic_type = self.tof.periodic_type
self.length_of_periodicity = len(self.tof.periodic_days)
self.minute_interval = minute_interval
self.window_interval = window_interval
self.tof.load_tof()
self.tof = self.tof.tof
self.regions = self.tof.keys()
self.spectrum_selection = 0
self.addition_technique = True
def get_tof_values(self, region):
"""
Obtain trajectory occurrence frequency values for a specific region.
"""
length = (self.window_interval/self.minute_interval) - 1
y = list()
region_tof = self.tof[region]
for day, hourly_tof in region_tof.iteritems():
mins = sorted(hourly_tof)
daily_y = list()
for i in mins:
daily_y.append(hourly_tof[i].get_occurrence_rate())
y.extend(daily_y[-length:] + daily_y[:-length])
return y
def reconstruct_tof_from_spectrum(self, region, addition_method=True, num_of_freqs=30):
"""
Reconstruct trajectory occurrence frequency values after they are transformed
into spectral dimensions.
"""
original = self.get_tof_values(region)
if addition_method:
spectrums, _ = self.get_significant_frequencies(original, num_of_freqs*2)
spectrums = spectrums[0:num_of_freqs]
else:
spectrums = self.get_highest_n_freq(fft(original), num_of_freqs)
reconstruction = 0
for spectrum in spectrums:
reconstruction += self.rectify_wave(
spectrum[2], spectrum[0], spectrum[1], len(original)
)
reconstruction = map(
lambda x: Lambda().get_occurrence_rate() if x <= 0.0 else x, reconstruction
)
return reconstruction, original
def model_selection(self, data, month, year, addition_method=True, max_freqs=30):
spectrum_selection = list()
validate_data = dict()
for region in self.regions:
validate_data.update({
region: trajectories_full_dates_periodic(
data[region], month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval
)
})
for num_of_freqs in range(1, max_freqs+1):
mse_region = list()
for region in self.regions:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, addition_method, num_of_freqs
)
mse_recon, _ = self._calculate_mse(
reconstruction_tof, original_tof, validate_data[region]
)
mse_region.append(mse_recon)
mse_region = [i for i in mse_region if i != -1]
spectrum_selection.append(sum(mse_region) / float(len(mse_region)))
spectrum_selection = spectrum_selection.index(min(spectrum_selection)) + 1
rospy.loginfo("Ideal total spectrums: %d" % spectrum_selection)
self.spectrum_selection = spectrum_selection
self.addition_technique = addition_method
def calculate_mse(self, region, data, month, year):
if self.spectrum_selection != 0:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique, self.spectrum_selection
)
else:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique
)
test_data = trajectories_full_dates_periodic(
data, month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval
)
reconstruction_tof = [i for j, i in enumerate(reconstruction_tof) if j % 5 == 0]
original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0]
test_data = [i for j, i in enumerate(test_data) if j % 5 == 0]
# Dropping -1 in test_data together with corresponding tof
test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data(
test_data, reconstruction_tof, original_tof
)
mse_recon = -1
mse_origin = -1
if len(reconstruction_tof) > 0 and len(original_tof) > 0:
mse_recon = mse(test_data, reconstruction_tof)
mse_origin = mse(test_data, original_tof)
rospy.loginfo("Calculated MSE for original tof Region %s: %.2f" % (region, mse_origin))
rospy.loginfo("Calculated MSE for reconstruction tof Region %s: %.2f" % (region, mse_recon))
# temp_recon = np.sqrt(mse_recon)
# sum_recon = 0
# for i in test_data:
# sum_recon += (i - temp_recon)**2
# print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1)))
# temp_recon = np.sqrt(mse_origin)
# sum_recon = 0
# for i in test_data:
# sum_recon += (i - temp_recon)**2
# print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1)))
return mse_recon, mse_origin
def rectify_wave(self, freq, amp, phs, num_of_points, up_thres=None, low_thres=None):
xf = np.linspace(0.0, num_of_points, num_of_points) # frequency varations
wave = amp * np.cos((freq * 2.0 * np.pi * xf) + phs)
for ind, val in enumerate(wave):
if low_thres is not None and val < low_thres:
wave[ind] = low_thres
if up_thres is not None and val > up_thres:
wave[ind] = up_thres
return wave
def get_significant_frequencies(self, data, total_freq=15, max_addition=10, max_iteration=1000):
N = len(data)
xf = np.linspace(0.0, N, N)
# initialise significant frequencies by taking frequency 0
spectrum_data = fft(data)
[amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 1)[0]
frequencies = [[amp, phs, freq]]
freq_occur_counter = {freq: 1}
exit_counter = 0
# data -= amp
while len(frequencies) < total_freq:
spectrum_data = fft(data)
# recreate wave of the highest frequency
[amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 2)[1]
if freq == 0:
[amp, phs, freq] = self.get_highest_n_freq(spectrum_data, 2)[0]
wave = amp * np.cos((freq * 2.0 * np.pi * xf) + phs)
# substracting data with the wave
data -= wave
if freq not in zip(*frequencies)[2]:
frequencies.append([amp, phs, freq])
freq_occur_counter.update({freq: 1})
else:
for ind, val in enumerate(frequencies):
if frequencies[ind][2] == freq and freq_occur_counter[freq] < max_addition:
frequencies[ind][0] += amp
frequencies[ind][1] = ((
freq_occur_counter[freq] * frequencies[ind][1]
) + phs) / (freq_occur_counter[freq] + 1)
freq_occur_counter[freq] += 1
exit_counter += 1
if exit_counter >= max_iteration:
break
return frequencies, data
def get_highest_n_freq(self, freqs, n=15):
N = len(freqs)
freqs = freqs[0:N/2]
indices = [i for i in range(len(freqs))]
angles = np.angle(freqs)
amplitudes = np.abs(freqs) / float(N)
sorted_result = sorted(zip(amplitudes, angles, indices), reverse=True)
n_freqs = sorted_result[:n]
return n_freqs
def _calculate_mse(self, reconstruction_tof, original_tof, test_data):
# reconstruction_tof = [i for j, i in enumerate(reconstruction_tof) if j % 5 == 0]
# original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0]
# test_data = [i for j, i in enumerate(test_data) if j % 5 == 0]
# Dropping -1 in test_data together with corresponding tof
test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data(
test_data, reconstruction_tof, original_tof
)
mse_recon = -1
mse_origin = -1
if len(reconstruction_tof) > 0 and len(original_tof) > 0:
mse_recon = mse(test_data, reconstruction_tof)
mse_origin = mse(test_data, original_tof)
return mse_recon, mse_origin
def _remove_unobserved_data(self, data1, data2, data3):
# Dropping -1 in data1, the corresponding indices in data2 and data3 are
# also dropped
deleted_indices = list()
for ind, trajs in enumerate(data1):
if trajs == -1:
deleted_indices.append(ind)
data1 = [j for i, j in enumerate(data1) if i not in deleted_indices]
data2 = [j for i, j in enumerate(data2) if i not in deleted_indices]
data3 = [j for i, j in enumerate(data3) if i not in deleted_indices]
return data1, data2, data3, deleted_indices
def prediction_accuracy(self, region, data, month, year, percentile=0.1, plot=False):
if self.spectrum_selection != 0:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique, self.spectrum_selection
)
else:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique
# region, False, 26
)
test_data = trajectories_full_dates_periodic(
data, month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval
)
original_predict = self._get_prediction(original_tof, test_data, percentile)
reconstr_predict = self._get_prediction(reconstruction_tof, test_data, percentile)
_, clean_ori_pred, clean_recon_pred, indices = self._remove_unobserved_data(
test_data, reconstr_predict, original_predict
)
if len(clean_ori_pred) and len(clean_recon_pred):
mse_predict = mse(clean_ori_pred, clean_recon_pred)
else:
mse_predict = -1
rospy.loginfo(
"Calculated MSE for prediction between original and reconstruction for Region %s: %.2f" % (region, mse_predict)
)
if plot:
for index in indices:
original_predict[index] = -1
reconstr_predict[index] = -1
x = np.linspace(0, len(test_data), len(test_data))
xticks = time_ticks(self.minute_interval, self.window_interval, self.periodic_type)
plt.plot(
x, original_predict, "-o", label="Prediction Original TOF"
)
plt.plot(
x, reconstr_predict, "-^", label="Prediction Reconstruction TOF"
)
plt.title("Prediction for Region %s" % region)
plt.xticks(x, xticks, rotation="vertical")
plt.xlabel("One Week Period with %d minutes interval and %d window time" % (self.minute_interval, self.window_interval))
plt.ylabel("Prediction (1=Anomalous, 0=Normal, -1=Unobserved)")
plt.ylim(ymin=-2, ymax=2)
plt.legend()
plt.show()
def _get_prediction(self, rates, data, percentile):
flag = list()
for ind, rate in enumerate(rates):
if rate > 0.0:
lower = poisson.ppf(percentile, rate)
upper = poisson.ppf(1-percentile, rate)
if data[ind] < lower or data[ind] > upper:
flag.append(1)
else:
flag.append(0)
else:
rospy.logwarn("Occurrence rate is %.2f" % rate)
flag.append(-1)
return flag
def plot_region_idft(self, region):
if self.spectrum_selection != 0:
y, data = self.reconstruct_tof_from_spectrum(
region, True, self.spectrum_selection
)
y2, _ = self.reconstruct_tof_from_spectrum(
region, False, self.spectrum_selection
)
else:
y, data = self.reconstruct_tof_from_spectrum(region, True)
y2, _ = self.reconstruct_tof_from_spectrum(region, False)
x = np.linspace(0, len(data), len(data))
xticks = time_ticks(self.minute_interval, self.window_interval, self.periodic_type)
plt.plot(
x, y, "-o", color="red", label="Amplitude Addition Reconstruction"
)
plt.plot(
x, y2, "-x", color="blue", label="Best Amplitude Reconstruction"
)
plt.plot(x, data, "-", color="green", label="Original Model")
plt.title("Reconstruction Occurrence Rate for Region %s" % region)
plt.xticks(x, xticks, rotation="vertical")
plt.xlabel("One Week Period with %d minutes interval and %d window time" % (self.minute_interval, self.window_interval))
plt.ylabel("Occurrence rate value")
plt.ylim(ymin=-5)
plt.legend()
plt.show()
class TOFPlot(object):
"""
This class is for plotting the trajectory occurrence frequency every n minute with
m window minute interval (as its bin) that is continuous starting from Monday 00:00
to Sunday 23:59
"""
def __init__(self, soma_map, soma_config, minute_interval=1, window_interval=10):
"""
Initialize plotting, it needs the name of the soma map, specific soma configuration,
the minute interval between two different occurrence rate, and the window interval that
acts as bins.
"""
self.tof = TrajectoryOccurrenceFrequencies(soma_map, soma_config, minute_interval, window_interval)
self.xticks = time_ticks(minute_interval, window_interval, self.tof.periodic_type)
self.x = np.arange(len(self.xticks))
self.minute_interval = minute_interval
self.window_interval = window_interval
self.periodic_length = len(self.tof.periodic_days)
self.tof.load_tof()
self.tof = self.tof.tof
self.regions = self.tof.keys()
self.colors = [
(0., 0., 0.), (0., 0., 1.), (0., 1., 0.), (0., 1., 1.),
(1., 0., 0.), (1., 0., 1.), (1., 1., 0.), (.75, .75, .75),
(0., 0., 0.), (0., 0., .5), (0., .5, 0.), (0., .5, .5),
(.5, 0., 0.), (.5, 0., .5), (.5, .5, 0.), (.25, .25, .25)
]
def get_y_yerr_per_region(self, region):
"""
Obtain the occurrence rate value, the mode of gamma distribution, for each region including
the lower percentile 0.025, and the upper percentile of the value 0.975 showing
the wide of the gamma distribution.
"""
length = (self.window_interval/self.minute_interval) - 1
y = list()
lower_percentile = list()
upper_percentile = list()
region_tof = self.tof[region]
for day, hourly_tof in region_tof.iteritems():
mins = sorted(hourly_tof)
daily_y = list()
daily_low = list()
daily_up = list()
for i in mins:
daily_y.append(hourly_tof[i].get_occurrence_rate())
daily_low.append(
abs(hourly_tof[i].get_occurrence_rate() - gamma.ppf(0.025, hourly_tof[i].occurrence_shape, scale=1/float(hourly_tof[i].occurrence_scale)))
# gamma.ppf(0.025, mins_tof[i].occurrence_shape, scale=1/float(mins_tof[i].occurrence_scale))
)
daily_up.append(
abs(hourly_tof[i].get_occurrence_rate() - gamma.ppf(0.975, hourly_tof[i].occurrence_shape, scale=1/float(hourly_tof[i].occurrence_scale)))
# gamma.ppf(0.975, mins_tof[i].occurrence_shape, scale=1/float(mins_tof[i].occurrence_scale))
)
y.extend(daily_y[-length:] + daily_y[:-length])
lower_percentile.extend(daily_low[-length:] + daily_low[:-length])
upper_percentile.extend(daily_up[-length:] + daily_up[:-length])
return y, lower_percentile, upper_percentile
def show_tof_per_region(self, region):
"""
Show the occurrence rate over a week/month for each region available from soma map
"""
y, low_err, up_err = self.get_y_yerr_per_region(region)
plt.errorbar(
self.x, y, yerr=[low_err, up_err], color='b', ecolor='r',
fmt="-o", label="Region " + region
# fmt="-o", label="Poisson Model"
)
# plt.title("Poisson Processes of the Corridor", fontsize=40)
plt.title("Occurrence Rate for Region %s" % region)
# plt.xticks(self.x, self.xticks, rotation="horizontal", fontsize=40)
plt.xticks(self.x, self.xticks, rotation="vertical")
plt.xlabel(
"One Week Period with %d minutes interval and %d window time" % (
self.minute_interval, self.window_interval
)
)
# plt.ylabel("Arrival Rate", fontsize=40)
plt.ylabel("Occurrence rate value")
plt.ylim(ymin=-1)
# plt.legend(prop={'size': 40})
plt.legend()
plt.show()
def show_tof(self):
"""
Show occurrence rate for all regions over a week/month
"""
for region in self.regions:
try:
color = self.colors[int(region) % len(self.colors)]
ecolor = self.colors[(int(region)**2 + 4) % len(self.colors)]
except:
color = (0., 0., 1.)
ecolor = (1., 0., 0.)
y, low_err, up_err = self.get_y_yerr_per_region(region)
plt.errorbar(
self.x, y, yerr=[low_err, up_err], color=color, ecolor=ecolor,
fmt="-o", label="Region " + region
)
plt.title("Occurrence Rate for All Regions")
plt.xticks(self.x, self.xticks, rotation="vertical")
plt.xlabel(
"One Week Period with %d minutes interval and %d window time" % (
self.minute_interval, self.window_interval
)
)
plt.ylabel("Occurrence rate value")
plt.ylim(ymin=-1)
plt.legend()
plt.show()
