def upload_osrm_binary(binaryfd, dbsession, merge=False, commit_every=1000): ''' Upload OSRM data into the db :param: binaryfd - a file descript pointing to the .osrm data :param: dbsession - an active :class:`sqlalchemy.Session` ''' insert_method = dbsession.add if not merge else dbsession.merge log.info("Loading OSRM node data") nodes_added = 0 for node in unpack_osrm_nodes(binaryfd): ormified = OSRMNode(node.id, node.lat, node.lon, node.bollard, node.traffic_light) insert_method(ormified) nodes_added += 1 if nodes_added % commit_every == 0: log.info("Added %i nodes", nodes_added) dbsession.commit() dbsession.commit() log.info("Added %i nodes", nodes_added) log.info("Loading OSRM edge data") existing_edges = set() edges_added = 0 for edge in unpack_osrm_edges(binaryfd): ormified = OSRMEdge( edge.node_a, edge.node_b, edge.distance, edge.weight, edge.bidirectional, ) # make sure we don't j key = OSRMEdge.hash_edge(edge.node_a, edge.node_b) if key in existing_edges: log.warning("Skipping existing edge: %s", repr(edge)) continue existing_edges.add(key) insert_method(ormified) edges_added += 1 if edges_added % commit_every == 0: log.info("Added %i edges", edges_added) dbsession.commit() dbsession.commit() log.info("Added %i edges", edges_added)
def test_unpack_nodes(): # Test node unpacking dummy_nodes, dummy_edges, dummy_binary = make_dummy_data(100, 20) new_nodes = list(unpack_osrm_nodes(dummy_binary)) eq_(len(new_nodes), 100) eq_(dummy_nodes[0], new_nodes[0])