def hydrate_POIs(roi, json_base_path):
for type_ in [PointOfInterest.TYPE_STOP, PointOfInterest.TYPE_PASS]:
result = []
for i in roi.poi_ids[type_]:
result.append(
utils.load_json(
os.path.join(json_base_path, ("poi_%s.json" % i)),
PointOfInterest))
roi.set_poi_list(result, type_)
def test_load_save_JSON(self):
ride = GpxParser(
'tests/data/sample_with_stop.gpx').get_ride_from_track()
p1 = ride.points[42]
filename = 'tests/data/test.json'
utils.save_json(filename, p1.to_JSON())
ok_(os.path.exists(filename))
js = utils.load_json(filename, GeoPoint)
eq_(js.lat, p1.lat)
os.remove(filename)
ok_(not os.path.exists(filename))
def lint(self):
# Loading ROIs
json_files = []
for f in os.listdir(self.workspace_folder):
if os.path.basename(f).startswith("roi_") and f.endswith('.json'):
json_files.append(os.path.join(self.workspace_folder, f))
ok = True
for jsf in json_files:
roi = utils.load_json(jsf, RegionOfInterest)
clen = len(roi.get_all_poi_coords())
RegionOfInterest.hydrate_POIs(roi, self.workspace_folder)
plen = len(roi.get_all_pois())
if clen != plen:
self.l.info("Problem with ROI: %s" % jsf)
ok = False
if ok:
self.l.info("All ROIs are OK! \o/")
else:
self.l.info("Problem to load ROI may affect the results.")
def generatemetrics(self):
# Loading ROIs
json_files = []
for f in os.listdir(self.workspace_folder):
if os.path.basename(f).startswith(
"roi_" + self.roi_version) and f.endswith('.json'):
json_files.append(os.path.join(self.workspace_folder, f))
ROIs = {}
pattern = re.compile("roi_(\d+_\d+_\d+)\.")
for jsf in json_files:
roi = utils.load_json(jsf, RegionOfInterest)
ROIs[pattern.search(jsf).group(1)] = roi
self.l.info("Loaded %d ROIs." % len(json_files))
self.l.info("Generating metrics...")
obj = Metrics(ROIs, self.workspace_folder).generate()
output = os.path.join(self.workspace_folder, "metrics.json")
utils.save_json(os.path.join(output), json.dumps(obj, indent=4))
self.l.info("Done! The metrics are available at %s" % output)
def detectpasses(self):
# Getting all gpx files in specified folder
gpx_files = []
for f in os.listdir(self.gpx_folder):
if f.endswith('.gpx'):
gpx_files.append(os.path.join(self.gpx_folder, f))
self.l.info("There's %d gpx files to be proccessed." % len(gpx_files))
# Loading ROIs
json_files = []
for f in os.listdir(self.workspace_folder):
if os.path.basename(f).startswith("roi_") and f.endswith('.json'):
json_files.append(os.path.join(self.workspace_folder, f))
ROIs = []
for jsf in json_files:
roi = utils.load_json(jsf, RegionOfInterest)
ROIs.append(roi)
self.l.info("Loaded %d ROIs." % len(json_files))
# Detecting Passes and storing Points of Interest
total = 0
total_stats = {
"# ROIs entered": 0,
"# ROIs stop POI": 0,
"# ROIs stop without POI": 0,
"# ROIs pass": 0,
"# ROIs pass but no cluster": 0,
"pass_speed_list": []
}
self.csv.info(
"\n--CSV-passes-data--\nride_file, ROI_in, ROI_stop_POI, ROI_stop_no_POI, ROI_pass_outside_cluster, ROI_pass, pass_speed_list"
)
for gpx in gpx_files:
ride = GpxParser(gpx).get_ride_from_track(
self.config.region_ignores)
passes, stats = detect_passes(ride, ROIs,
self.config.dbscan_eps_in_meters,
self.config.dbscan_min_samples,
self.workspace_folder)
for k in stats.keys():
total_stats[k] += stats[k]
passes_count = len(passes)
total += passes_count
self.csv.info(
"%s,%3d,%3d,%3d,%3d,%3d,%s" %
(os.path.basename(gpx), stats["# ROIs entered"],
stats["# ROIs stop POI"], stats["# ROIs stop without POI"],
stats["# ROIs pass but no cluster"], passes_count, " ".join(
map(str, stats['pass_speed_list']))))
for p in passes:
utils.save_json(
os.path.join(self.workspace_folder, "poi_%s.json" % p.id),
p.to_JSON())
self.csv.info("--CSV--\n")
self.csv.info("\n--CSV-pass-speeds--")
self.csv.info(",".join(map(str, total_stats['pass_speed_list'])))
self.csv.info("--CSV--\n")
self.l.info(
"Done! There was %d passes detected\nThe data is available at %s" %
(total, self.workspace_folder))
def clusterize(self):
import numpy
import commutemate.clustering as clustering
json_files = []
for f in os.listdir(self.workspace_folder):
if os.path.basename(f).startswith("poi_") and f.endswith('.json'):
json_files.append(os.path.join(self.workspace_folder, f))
self.l.info("There's %d POI(s) to be clusterized. Preparing data..." %
len(json_files))
geo_coords = []
POIs = []
for jsf in json_files:
poi = utils.load_json(jsf, PointOfInterest)
POIs.append(poi)
geo_coords.append([poi.point.lat, poi.point.lon])
POIs = numpy.array(POIs)
X = numpy.array(geo_coords)
self.l.info(
"Running DBSCAN with eps=%d meters, min_samples=%d, metric=%s" %
(self.config.dbscan_eps_in_meters, self.config.dbscan_min_samples,
'haversine'))
# ============== clustering with dbscan =============== #
db = clustering.cluster_with_bearing_weight(
POIs, X, self.config.dbscan_eps_in_meters,
self.config.dbscan_min_samples)
self.l.info("Done!")
self.l.info("Creating regions of interest")
labels = db #.labels_
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
# ============== creating ROIs =============== #
ROIs = clustering.create_ROIs(POIs, labels, self.workspace_folder,
self.config.dbscan_eps_in_meters)
self.csv.info(
"\n--CSV-ROI-data--\ncenter_lat, center_long, range_meters, POI_count, bearing_avg, bearing_std"
)
for roi_ in ROIs:
self.csv.info(
"%11.7f,%11.7f,%3d,%3d,%6.2f,%6.2f" %
(roi_.center_range[0], roi_.center_range[1],
roi_.center_range[2], len(roi_.get_all_poi_coords()),
roi_.bearing_avg, roi_.bearing_std))
self.csv.info("--CSV--\n")
self.l.info(
"Done! There was %d regions of interest detected\nThe data is available at %s"
% (len(ROIs), self.workspace_folder))
# ============== rendring map =============== #
self.l.info("Rendering visualization")
o = clustering.render_map(ROIs, POIs, X, labels, self.workspace_folder,
self.config.dbscan_eps_in_meters)
self.l.info("Done!\nThe map visualization is available at %s" % o)
def __load_POI(self, id):
poi = self.POIs.get(id, None)
if not poi:
poi = utils.load_json(os.path.join(self.workspace_folder, "poi_%s.json" % id), PointOfInterest)
self.POIs[id] = poi
return poi
def detect_passes(ride, ROIs, eps_in_meters, min_samples, workspace_folder):
import numpy
import commutemate.clustering as clustering
stops = []
on_a_stop = False
stop_buffer = None
previous_stop = None
passes = []
on_a_roi = False
pass_buffer = []
current_roi = None
previous_pass = None
stats = {
"# ROIs entered": 0,
"# ROIs stop POI": 0,
"# ROIs stop without POI": 0,
"# ROIs pass": 0,
"# ROIs pass but no cluster": 0,
"pass_speed_list": []
}
for p in ride.points[1:]:
if not current_roi:
roi = __inside_a_ROI(p, ROIs)
elif current_roi and utils.is_inside_range(current_roi.center_range,
p):
roi = current_roi
else:
roi = None
if roi:
on_a_roi = True
if not current_roi:
stats["# ROIs entered"] += 1
current_roi = roi
if p.speed < STOPPED_SPEED_KMH_THRESHOLD:
on_a_stop = True
stop_buffer = p
else:
pass_buffer.append(p)
else:
if on_a_stop:
poi = PointOfInterest(stop_buffer, PointOfInterest.TYPE_STOP,
ride.origin, ride.destination)
if current_roi.is_poi_included(poi.id):
# Updating POI with preivous ROIs info
poi = utils.load_json(
os.path.join(workspace_folder, "poi_%s.json" % poi.id),
PointOfInterest)
poi.previous_stop_ROI = previous_stop.id if previous_stop else None
poi.previous_pass_ROI = previous_pass.id if previous_pass else None
utils.save_json(
os.path.join(workspace_folder, "poi_%s.json" % poi.id),
poi.to_JSON())
previous_stop = poi
stats["# ROIs stop POI"] += 1
else:
stats["# ROIs stop without POI"] += 1
on_a_stop = False
stop_buffer = None
on_a_roi = False
pass_buffer = []
current_roi = None
elif on_a_roi:
pass_in_cluster = []
# check from all the points inside a ROI which ones are inside the original cluster
for ppass in pass_buffer:
poi = PointOfInterest(ppass, PointOfInterest.TYPE_PASS,
ride.origin, ride.destination)
poi.set_duration(0)
poi.set_previous_stop(previous_stop)
poi.previous_stop_ROI = previous_stop.id if previous_stop else None
poi.previous_pass_ROI = previous_pass.id if previous_pass else None
# need to hydrate ROI to have POIs bearing info
RegionOfInterest.hydrate_POIs(current_roi,
workspace_folder)
current_roi.set_poi_list([poi], PointOfInterest.TYPE_PASS)
POIs = numpy.array(current_roi.get_all_pois())
X = numpy.array(current_roi.get_all_poi_coords())
# If pass point is not part of stop cluster, this means that the pass is in another direction
db = clustering.cluster_with_bearing_weight(
POIs, X, eps_in_meters, min_samples)
n_clusters_ = len(set(db))
if n_clusters_ == 1:
pass_in_cluster.append(poi)
current_roi.set_poi_list([], PointOfInterest.TYPE_PASS)
if len(pass_in_cluster) > 0:
# we have officially a pass, the crowd goes crazy
stats["# ROIs pass"] += 1
poi = pass_in_cluster[
len(pass_in_cluster) /
2] # get buffer mid point as point for POI
stats["pass_speed_list"].append(poi.point.speed)
previous_pass = poi
passes.append(poi)
else:
stats["# ROIs pass but no cluster"] += 1
on_a_stop = False
stop_buffer = None
on_a_roi = False
pass_buffer = []
current_roi = None
return passes, stats