def convert_osmtogeojson(filename):
_get_all_coordinates(filename)
feature_collection = []
for item in parse_file(filename): # generator!!
if isinstance(item, Node) and item.tags != {}:
point = Point(dict_of_coords[item.id])
feature = Feature(geometry=point, id=item.id, properties=item.tags)
feature_collection.append(feature)
elif isinstance(item, Way) and 'highway' in item.tags:
coords = _get_coords_of_edge(item.nodes)
line_string = LineString(coords)
feature = Feature(geometry=line_string,
id=item.id,
properties=item.tags)
feature_collection.append(feature)
print_info("\rParsing features - feature count: {:,}".format(
len(feature_collection)).replace(",", " "),
end='')
print("")
geojson_file = FeatureCollection(feature_collection)
# with open('data/output.geojson', 'w') as outfile:
# geojson.dump(geojson_file, outfile)
# outfile.close()
return geojson_file
def save_csv(data: List[List[str]], filepath: str, append: bool = False):
mode = 'a' if append else 'w'
print_info("Saving csv file to: {}".format(os.path.realpath(filepath)))
with open(filepath, mode, newline='') as csvfile:
writer = csv.writer(csvfile)
for row in data:
writer.writerow(row)
def get_cleaned_geojson(json_dict):
print_info("Cleaning geojson")
prune_geojson_file(json_dict)
print_info("Removing empty features")
json_dict['features'] = [i for i in json_dict["features"]
if i] # remove empty dicts
return json_dict
def get_nearest_edge(self, point: Point):
search_bounds = Point(point).buffer(self.search_size).bounds
candidates = self.index.intersection(search_bounds, objects='raw')
min_distance = sys.maxsize
nearest = None
for candidate in candidates:
edge: LinestringEdge = candidate
distance = point.distance(edge.linestring)
if distance < min_distance:
min_distance = distance
nearest = edge
if not nearest:
print_info("No edge found in specified distance ({} m).".format(
self.search_size))
envelope = Polygon(
((search_bounds[0], search_bounds[3]), (search_bounds[2],
search_bounds[3]),
(search_bounds[2], search_bounds[1]), (search_bounds[0],
search_bounds[1])))
if not envelope.intersects(nearest.linestring):
print_info("solution does not have to be exact")
return nearest
def clean_geojson_files(input_file_path: str = config.geojson_file,
output_file_path: str = config.cleaned_geojson_file,
keep_attributes: Set[str] = SET_OF_USEFUL_PROPERTIES,
remove_attributes: Set[str] = None):
print_info('Cleaning geoJSON - input file: {}, cleaned file: {}'.format(
input_file_path, output_file_path))
start_time = time.time()
feature_collection = roadmaptools.inout.load_geojson(input_file_path)
edges = [
item for item in feature_collection['features']
if item['geometry']['type'] == 'LineString'
]
# global nonempty_columns
# nonempty_columns = get_non_empty_columns(edges)
prune_geojson_file(feature_collection, edges, keep_attributes,
remove_attributes)
# print_info("Removing empty features")
# json_dict['features'] = [i for i in json_dict["features"] if i] # remove empty dicts
print_info('Cleaning complete. (%.2f secs)' % (time.time() - start_time))
roadmaptools.inout.save_geojson(feature_collection, output_file_path)
def load_pickle(filepath: str):
print_info("Loading pickle file from: {}".format(
os.path.realpath(filepath)))
with open(filepath, 'rb') as pickled_data:
data = pickle.load(pickled_data)
return data
def load_from_geojson(geojson: FeatureCollection):
# projection determination
first_coord = geojson['features'][0]['geometry']['coordinates']
projection = roadmaptools.utm.TransposedUTM(first_coord[1], first_coord[0])
print_info("Projection determined from the first coordinate: {}{}".format(
projection.origin_zone_number, projection.origin_zone_letter))
CoordinateConvertor.projection = projection
print_info("Creating networkx graph from geojson")
for item in tqdm(geojson['features'], desc="processing features"):
if item["geometry"]["type"] == "LineString":
coords = item['geometry']['coordinates']
coord_from = roadmaptools.utm.wgs84_to_utm(coords[0][1],
coords[0][0],
projection)
coord_to = roadmaptools.utm.wgs84_to_utm(coords[-1][1],
coords[-1][0], projection)
node_from = _get_node(coord_from[0], coord_from[1])
node_to = _get_node(coord_to[0], coord_to[1])
edge = LinestringEdge(item, CoordinateConvertor.convert, node_from,
node_to)
# TODO legacy, remove after moving id from properties to id attribute
edge_id = item['properties']['id'] if "id" in item[
'properties'] else item['id']
length = item['properties']['length'] if 'length' in item['properties'] \
else roadmaptools.geometry.get_distance(coord_from, coord_to)
graph.add_edge(node_from,
node_to,
id=edge_id,
length=length,
edge=edge)
def _get_all_coordinates(filename):
for item in parse_file(filename): # generator!!
if isinstance(item, Node):
dict_of_coords[item.id] = (item.lon, item.lat)
print_info("\rParsing all coordinates - node count: {:,}".format(
len(dict_of_coords)).replace(",", " "),
end='')
print("")
def load_csv_to_pandas(filepath: str,
delimiter: str = ",",
header: List[str] = None) -> pandas.DataFrame:
print_info("Loading csv file from: {} to dataframe".format(
os.path.realpath(filepath)))
if header:
return pandas.read_csv(filepath, names=header)
return pandas.read_csv(filepath)
def download_file(url: str, file_name: str):
print_info("Downloading file from {} to {}".format(url, file_name))
with Progressbar(unit='B',
unit_scale=True,
miniters=1,
desc="Downloading file") as progressbar:
urllib.request.urlretrieve(url, file_name, progressbar.update_to)
print_info("Download finished")
def filter_geojson_features_by_graph(geojson_data: FeatureCollection,
edge_ids: set):
print_info("Filtering geojson by edge id set")
new_features = []
for item in tqdm(geojson_data['features'], desc="processing features"):
if item['properties']['id'] in edge_ids:
new_features.append(item)
geojson_data["features"] = new_features
def clean_geojson_files(input_file_path: str = config.geojson_file, output_file_path: str = config.cleaned_geojson_file,
keep_attributes: Set[str] = SET_OF_USEFUL_PROPERTIES, remove_attributes: Set[str] = None):
print_info('Cleaning geoJSON - input file: {}, cleaned file: {}'.format(input_file_path, output_file_path))
start_time = time.time()
feature_collection = roadmaptools.inout.load_geojson(input_file_path)
prune_geojson_file(feature_collection, keep_attributes, remove_attributes)
print_info('Cleaning complete. (%.2f secs)' % (time.time() - start_time))
roadmaptools.inout.save_geojson(feature_collection, output_file_path)
def _save_map_for_ap():
print_info('Preparing files for agentpolis-demo... ', end='')
start_time = time.time()
geojson_file = roadmaptools.inout.load_geojson(
config.file_with_computed_parameters_filepath)
edges, nodes = roadmaptools.prepare_geojson_to_agentpolisdemo.get_nodes_and_edges_for_agentpolisdemo(
geojson_file)
roadmaptools.inout.save_geojson(nodes, config.nodes_filepath)
roadmaptools.inout.save_geojson(edges, config.edges_filepath)
print_info('done. (%.2f secs)' % (time.time() - start_time))
def filter_osm_file():
""" Downloads (and compiles) osmfilter tool from web and
calls that osmfilter to only filter out only the road elements.
"""
print_info('Filtering OSM file...')
start_time = time.time()
if check_osmfilter():
# params = '--keep="highway=motorway =motorway_link =trunk =trunk_link =primary =primary_link =secondary' \
# ' =secondary_link =tertiary =tertiary_link =unclassified =unclassified_link =residential =residential_link' \
# ' =living_street" --drop="access=no"'
params = config.osm_filter_params
command = './osmfilter' if platform.system(
) == 'Linux' else 'osmfilter.exe'
if platform.system() == 'Linux':
filter_command = '%s "%s" %s | pv > "%s"' % (
command, config.osm_map_filename, params,
config.filtered_osm_filename)
else:
filter_command = '%s "%s" %s > "%s"' % (
command, config.osm_map_filename, params,
config.filtered_osm_filename)
print_info("Running the osmfilter: {}".format(filter_command))
os.system(filter_command)
else:
print_info('Osmfilter not available. Exiting.')
exit(1)
print_info('Filtering finished. (%.2f secs)' % (time.time() - start_time))
def google_maps_find_path(self, start: Tuple[float, float], target: Tuple[float, float], time: datetime,
model: str= "best_guess"):
try:
gmaps = googlemaps.Client(key='AIzaSyDoCeLZhFJkx2JTLH8UMcsouaVUIwbV_wY')
# time = datetime(2017, 11, 14, 7, 0, 0) # 7 hours after midnight
# print_info("Request sent")
result = gmaps.directions(start, target, mode="driving", language="en-GB", units="metric",
departure_time=time, traffic_model=model)
# print_info("Response obtained")
# print(self.get_velocity(result))
# duration = result['rows'][0]['elements'][0]['duration_in_traffic']['value']
# distance = result['rows'][0]['elements'][0]['distance']['value']
return result
except (googlemaps.exceptions) as error:
print_info("Google maps exception: {}".format(error))
def _compute_edge_parameters():
print_info('Estimating travel speed (using max speed)... ', end='')
start_time = time.time()
input_file = roadmaptools.inout.load_geojson(config.sanitized_filepath)
feature_collection_with_speeds = roadmaptools.estimate_speed_from_osm.get_geojson_with_speeds(
input_file)
# points = roadmaptools.export_nodes_and_id_maker.export_points_to_geojson(feature_collection_with_speeds)
# feature_collection_with_speeds_and_ids \
# = roadmaptools.export_nodes_and_id_maker.get_geojson_with_unique_ids(feature_collection_with_speeds)
roadmaptools.inout.save_geojson(
feature_collection_with_speeds,
config.file_with_computed_parameters_filepath)
print_info('done. (%.2f secs)' % (time.time() - start_time))
def get_biggest_component(graph: DiGraph) -> set:
biggest_subgraph = graph
if not nx.is_strongly_connected(graph):
maximum_number_of_nodes = -1
for subgraph in nx.strongly_connected_components(graph):
if len(subgraph) > maximum_number_of_nodes:
maximum_number_of_nodes = len(subgraph)
biggest_subgraph = subgraph
id_dict = nx.get_edge_attributes(graph, "id")
print_info("Creating edge id set")
edge_ids = set()
for coordinates, id in id_dict.items():
if coordinates[0] in biggest_subgraph and coordinates[
1] in biggest_subgraph:
edge_ids.add(id)
return edge_ids
def compute_edge_curvatures(input_filename: str, output_filename: str):
print_info('Computing average edge curvatures.')
start_time = time.time()
input_stream = open(input_filename, encoding='utf8')
output_stream = open(output_filename, 'w')
print_info("Loading geojson from: {}".format(input_filename))
geojson_file = load_geojson(input_stream)
print_info("Calculating curvature")
geojson_out = get_geojson_with_curvature(geojson_file)
print_info("Saving geojson to: {}".format(output_filename))
save_geojson(geojson_out, output_stream)
input_stream.close()
output_stream.close()
print_info('Curvature computation process finished. (%.2f secs)' % (time.time() - start_time))
def _build_index(road_graph: RoadGraph, path: str = None):
if path:
cache_ready = os.path.isfile(path + ".idx")
idx = index.Index(path)
else:
cache_ready = False
idx = index.Index()
if not cache_ready:
print_info("Creating R-tree from geojson roadmap")
for from_node, to_node, data in tqdm(
road_graph.graph.edges(data=True),
desc="processing edges"):
edge: LinestringEdge = data["edge"]
# data["attr"]["from"] = from_node
# data["attr"]["to"] = to_node
idx.insert(data["id"], edge.linestring.bounds, edge)
if path:
idx.close()
idx = index.Index(path)
return idx
def estimate_posted_speed(input_filename: str, output_filename: str):
print_info('Estimating travel speed')
start_time = time.time()
input_stream = codecs.open(input_filename, encoding='utf8')
output_stream = open(output_filename, 'w')
print_info("Loading file from: {}".format(input_filename))
geojson_file = load_geojson(input_stream)
print_info("Computing speed")
geojson_out = get_geojson_with_speeds(geojson_file)
print_info("Saving file to: {}".format(output_filename))
save_geojson(geojson_out, output_stream)
input_stream.close()
output_stream.close()
print_info('Speed estimation completed. (%.2f secs)' %
(time.time() - start_time))
def check_osmfilter():
# determine if osmfilter is installed, otherwise download it
print_info("Checking if osmfilter is installed.")
my_platform = platform.system(
) # get system info. Values: 'Linux', 'Windows'
if my_platform == 'Linux': # check if osmfilter is downloaded
executable = 'osmfilter'
if not os.path.exists(executable):
print_info('Downloading and compiling osmfilter... ')
download_file('http://m.m.i24.cc/osmfilter.c', 'osmfilter.c')
os.system('cc -x c osmfilter.c -O3 -o osmfilter')
return True
elif my_platform == 'Windows':
executable = 'osmfilter.exe'
if not os.path.exists(executable):
print_info('Downloading and compiling osmfilter... ')
download_file('http://m.m.i24.cc/osmfilter.exe', 'osmfilter.exe')
return True
else:
print_err('OSM filtering not implemented for platform: %s. ' %
my_platform)
return False
def download_cities(bounding_boxes: List[Tuple[float, float, float, float]],
filepath: str):
"""
Downloads osm map and saves it as .geojson file.
:param bounding_boxes: Order of coordinates in bounding box: (min lat, min lon, max lan, max lon)
:param filepath: path to output file
:return:
"""
print_info("Downloading map from Overpass API")
api = overpass.API(debug=True, timeout=600)
query = '(('
for bounding_box in bounding_boxes:
if bounding_box[0] >= bounding_box[2] or bounding_box[
1] >= bounding_box[3]:
raise Exception('Wrong order in: ', bounding_box)
query += 'way({})[{}][access!="no"];'.format(
",".join(map(str, list(bounding_box))), HIGHWAY_FILTER)
query += ')->.edges;.edges >->.nodes;);'
out = api.get(query, verbosity='geom')
roadmaptools.inout.save_geojson(out, filepath)
def load_graph(
data: geojson.feature.FeatureCollection,
attribute_constructor: Callable[[geojson.LineString], Dict] = None,
coordinate_convertor: Callable[[float, float], Tuple[float, float]] = None
) -> nx.DiGraph:
g = nx.DiGraph()
print_info("Creating networkx graph from geojson")
for item in tqdm(data['features'], desc="processing features"):
coords = item['geometry']['coordinates']
if coordinate_convertor:
coord_from = coordinate_convertor(coords[0][1], coords[0][0])
coord_to = coordinate_convertor(coords[-1][1], coords[-1][0])
else:
coord_from = (coords[0][1], coords[0][0])
coord_to = (coords[-1][1], coords[-1][0])
if attribute_constructor:
g.add_edge(coord_from,
coord_to,
id=item['properties']['id'],
attr=attribute_constructor(item))
else:
g.add_edge(coord_from, coord_to, id=item['properties']['id'])
return g
def load_from_geojson(self, geojson_filepath: str):
if self.use_cache and os.path.exists(self.cache_filepath):
self.graph = networkx.read_gpickle(self.cache_filepath)
self.projection = roadmaptools.utm.TransposedUTM.from_zone(self.graph.graph["zone_number"],
self.graph.graph["zone_letter"])
CoordinateConvertor.projection = self.projection
print_info("Projection loaded from the cache: {}{}".format(
self.projection.origin_zone_number, self.projection.origin_zone_letter))
else:
geojson = roadmaptools.inout.load_geojson(geojson_filepath)
# projection determination
for item in geojson['features']:
if item["geometry"]["type"] == "LineString":
first_coord = geojson['features'][0]['geometry']['coordinates'][0]
break
self.projection = roadmaptools.utm.TransposedUTM.from_gps(first_coord[1], first_coord[0])
print_info("Projection determined from the first coordinate: {}{}".format(
self.projection.origin_zone_number, self.projection.origin_zone_letter))
CoordinateConvertor.projection = self.projection
self.graph = DiGraph(zone_number=self.projection.origin_zone_number,
zone_letter=self.projection.origin_zone_letter)
edge_counter = 0
print_info("Creating networkx graph from geojson")
for item in tqdm(geojson['features'], desc="processing features"):
if item["geometry"]["type"] == "LineString":
coords = item['geometry']['coordinates']
coord_from = roadmaptools.utm.wgs84_to_utm(coords[0][1], coords[0][0], self.projection)
coord_to = roadmaptools.utm.wgs84_to_utm(coords[-1][1], coords[-1][0], self.projection)
node_from = self._get_node(coord_from[0], coord_from[1])
node_to = self._get_node(coord_to[0], coord_to[1])
edge = LinestringEdge(item, CoordinateConvertor.convert, node_from, node_to)
# TODO legacy, remove after moving id from properties to id attribute
edge_id = item['properties']['id'] if "id" in item['properties'] else item['id']
length = item['properties']['length'] if 'length' in item['properties'] \
else roadmaptools.geometry.get_distance(coord_from, coord_to)
if node_from in self.graph and node_to in self.graph[node_from]:
a = 1
self.graph.add_edge(node_from, node_to, id=edge_id, length=length, edge=edge)
edge_counter += 1
if edge_counter != len(self.graph.edges):
a = 1
if self.use_cache:
networkx.write_gpickle(self.graph, self.cache_filepath)
def convert_osm_to_geojson():
input_file = config.filtered_osm_filename
output_file = config.geojson_file
print_info(
'Converting from OSM format to geoJSON - input file: {}, output file: {}'
.format(input_file, output_file))
start_time = time.time()
geojson_file = convert_osmtogeojson(input_file)
f = open(output_file, 'w')
save_geojson(geojson_file, f)
print_info("Geojson saved successfully to {}".format(output_file))
f.close()
print_info('Converting from OSM to geoJSON finished. (%.2f secs)' %
(time.time() - start_time))
def simplify_geojson(input_file=config.sanitized_geojson_file,
output_file=config.simplified_file):
print_info('Simplifying geoJSON')
start_time = time.time()
# l_check set True whether you don't want to simplify edges with different number of lanes
# c_check set True whether you don't want to simplify edges with different curvature
geojson_file = roadmaptools.inout.load_geojson(input_file)
print_info("Simplification process started")
geojson_out = get_simplified_geojson(geojson_file,
l_check=False,
c_check=False)
print_info('Simplification completed. (%.2f secs)' %
(time.time() - start_time))
roadmaptools.inout.save_geojson(geojson_out, output_file)
def extract_file(filename: str):
new_filename = filename.replace(".bz2", "")
compressed_size = os.path.getsize(filename)
print_info("Extracting file {} to {} (compressed size: {})".format(
filename, new_filename, compressed_size))
block_size = 100 * 1024
uncompressed_size = 0
with open(new_filename, 'wb') as new_file, bz2.BZ2File(filename,
'rb') as file:
for data in iter(lambda: file.read(block_size), b''):
new_file.write(data)
uncompressed_size += block_size
print_info("\rDecompressing - decompressed size: {:,}B".format(
uncompressed_size).replace(",", " "),
end='')
uncompressed_size = os.path.getsize(new_filename)
print_info("\nExtraction finished: uncompressed size: {:,}B".format(
uncompressed_size).replace(",", " "))
import roadmaptools.inout
from roadmaptools.printer import print_info
# from roadmaptools.init import config
print_info("Loading start")
roadmaptools.inout.load_gpx(
"/home/fido/AIC data/Shared/EXPERIMENTAL/traces/traces-raw.gpx")
print_info("Loading end")
def load_geojson(filepath: str) -> geojson.feature.FeatureCollection:
print_info("Loading geojson file from: {}".format(
os.path.realpath(filepath)))
input_stream = open(filepath, encoding='utf8')
json_dict = geojson.load(input_stream)
return json_dict
id=edge_id,
length=length,
edge=edge)
def _get_node(x: float, y: float) -> Node:
id = roadmaptools.utm.get_id_from_utm_coords(x, y)
if id in node_map:
return node_map[id]
else:
node = _create_node(x, y, id)
node_map[id] = node
return node
def _create_node(x: float, y: float, id: int) -> Node:
return Node(x, y, id)
print_info("START STNDARD")
map = roadmaptools.inout.load_geojson(
r"C:\AIC data\Shared\Map Matching Benchmark\2015 100 tracks dataset\00000043/map.geojson"
)
load_from_geojson(map)
print_info("END STANDARD")
networkx.write_gpickle(graph, "pi.pickle")
print_info("START_PICKLE")
networkx.read_gpickle("pi.pickle")
print_info("END PICKLE")