def get_schedule_source(self, refresh=False):
"""Loads the schedule source information.
Loads a schedule source file from either a path or a url specified
in the config file
:return schedule_source: The schedule read from a file.
"""
if 'schedule_source' not in self.data:
return None
else:
source_file = self.data['schedule_source']
cached_file = self.data['selector'] + '-schedule'
# Preferably return cached data about schedule
if refresh is False:
# Check if _schedule_source data is already present
if not self._schedule_source:
# If not, try to get _schedule_source from file cache
self._schedule_source = Cache.read_file(cached_file)
# Return cached data if found
if bool(self._schedule_source):
return self._schedule_source
# No cached data was found or refresh was forced
print("Load schedule source information from " + source_file)
# Check if local file exists
if os.path.isfile(source_file):
# Open file and add to config object
with open(source_file, 'r') as f:
schedule_source = f.read()
else:
# Check if it is a valid url
try:
schedule_source_file = urlopen(source_file)
except ValueError:
sys.stderr.write(
"Error: Couldn't find schedule_source file.\n")
sys.exit(0)
schedule_source = schedule_source_file.read()
self._schedule_source = schedule_source
# Cache data
Cache.write_file(cached_file, self._schedule_source)
return self._schedule_source
def get_schedule_source(self, refresh=False):
"""Loads the schedule source information.
Loads a schedule source file from either a path or a url specified
in the config file
:return schedule_source: The schedule read from a file.
"""
if 'schedule_source' not in self.data:
return None
else:
source_file = self.data['schedule_source']
cached_file = self.data['selector'] + '-schedule'
# Preferably return cached data about schedule
if refresh is False:
# Check if _schedule_source data is already present
if not self._schedule_source:
# If not, try to get _schedule_source from file cache
self._schedule_source = Cache.read_file(cached_file)
# Return cached data if found
if bool(self._schedule_source):
return self._schedule_source
# No cached data was found or refresh was forced
logging.info("Load schedule source information from %s", source_file)
# Check if local file exists
if os.path.isfile(source_file):
# Open file and add to config object
with open(source_file, 'r') as f:
schedule_source = f.read()
else:
# Check if it is a valid url
try:
schedule_source_file = urlopen(source_file)
except ValueError:
logging.error("Couldn't find schedule_source file.")
sys.exit(0)
schedule_source = schedule_source_file.read()
self._schedule_source = schedule_source
# Cache data
Cache.write_file(cached_file, self._schedule_source)
return self._schedule_source
def test_refresh_routes_cache(self):
data = OsmConnector(self.config)
cache_file = os.path.join(
self.standard_variables['data_dir'], self.selector + "-routes.pkl")
mocked_overpass_data_file = self.standard_variables['mocked_overpass_routes']
if os.path.isfile(cache_file):
os.remove(cache_file)
with patch("osm2gtfs.core.osm_connector.OsmConnector._query_routes") as mocked1:
overpass_xml = open(mocked_overpass_data_file, mode='r').read()
api = overpy.Overpass()
mocked1.return_value = api.parse_xml(overpass_xml)
data.get_routes(refresh=True)
self.assertTrue(os.path.isfile(cache_file), 'The routes cache file creation failed')
cache = Cache()
routes = cache.read_data(self.selector + "-routes")
self.assertEqual(
len(routes), self.required_variables['routes_count'],
'Wrong count of routes in the cache file')
def _get_names_for_unnamed_stops(self):
"""Intelligently guess stop names for unnamed stops by sourrounding
street names and amenities.
Caches stops with newly guessed names.
"""
# Loop through all stops
for stop in self.stops['regular'].values():
# If there is no name, query one intelligently from OSM
if stop.name == "[" + self.stop_no_name + "]":
self._find_best_name_for_unnamed_stop(stop)
logging.info("* Found alternative stop name: " + stop.name +
" - " + stop.osm_url)
# Cache stops with newly created stop names
Cache.write_data(self.selector + '-stops', self.stops)
def test_refresh_routes_cache(self):
data = OsmConnector(self.config)
cache_file = os.path.join(
self.standard_variables['data_dir'], self.selector + "-routes.pkl")
mocked_overpass_data_file = self.standard_variables['mocked_overpass_routes']
if os.path.isfile(cache_file):
os.remove(cache_file)
with patch("osm2gtfs.core.osm_connector.OsmConnector._query_routes") as mocked1:
overpass_xml = open(mocked_overpass_data_file, mode='r').read()
api = overpy.Overpass()
mocked1.return_value = api.parse_xml(overpass_xml)
data.get_routes(refresh=True)
self.assertTrue(os.path.isfile(cache_file), 'The routes cache file creation failed')
cache = Cache()
routes = cache.read_data(self.selector + "-routes")
self.assertEqual(
len(routes), self.required_variables['routes_count'],
'Wrong count of routes in the cache file')
def test_refresh_stops_cache(self):
data = OsmConnector(self.config)
cache_file = os.path.join(self.standard_variables['data_dir'], self.selector + "-stops.pkl")
mocked_overpass_data_file = self.standard_variables['mocked_overpass_stops']
if os.path.isfile(cache_file):
os.remove(cache_file)
with patch("osm2gtfs.core.osm_connector.OsmConnector._query_stops") as mocked1:
overpass_xml = open(mocked_overpass_data_file, mode='r').read()
api = overpy.Overpass()
mocked1.return_value = api.parse_xml(overpass_xml)
data.get_stops(refresh=True)
self.assertTrue(os.path.isfile(cache_file), 'The stops cache file creation failed')
cache = Cache()
stops = cache.read_data(self.selector + "-stops")
amount_of_stops = len(stops['regular']) + len(stops['stations'])
print("> Amount of osm stops: " + str(amount_of_stops))
self.assertEqual(
amount_of_stops, self.required_variables['stops_osm_count'],
'Wrong count of stops in the cache file')
def test_refresh_routes_cache(self):
data = OsmConnector(self.config)
cache_file = os.path.join(self.data_dir, "accra-routes.pkl")
mocked_overpass_data_file = os.path.join(self.fixture_folder,
"overpass-routes.xml")
if os.path.isfile(cache_file):
os.remove(cache_file)
with patch("osm2gtfs.core.osm_connector.OsmConnector._query_routes"
) as mocked1:
overpass_xml = open(mocked_overpass_data_file, mode='r').read()
api = overpy.Overpass()
mocked1.return_value = api.parse_xml(overpass_xml)
data.get_routes(refresh=True)
self.assertTrue(os.path.isfile(cache_file),
'The routes cache file creation failed')
cache = Cache()
routes = cache.read_data('accra-routes')
self.assertEqual(len(routes), 277,
'Wrong count of routes in the cache file')
def test_refresh_stops_cache(self):
data = OsmConnector(self.config)
cache_file = os.path.join(self.standard_variables['data_dir'], self.selector + "-stops.pkl")
mocked_overpass_data_file = self.standard_variables['mocked_overpass_stops']
if os.path.isfile(cache_file):
os.remove(cache_file)
with patch("osm2gtfs.core.osm_connector.OsmConnector._query_stops") as mocked1:
overpass_xml = open(mocked_overpass_data_file, mode='r').read()
api = overpy.Overpass()
mocked1.return_value = api.parse_xml(overpass_xml)
data.get_stops(refresh=True)
self.assertTrue(os.path.isfile(cache_file), 'The stops cache file creation failed')
cache = Cache()
stops = cache.read_data(self.selector + "-stops")
amount_of_stops = len(stops['regular']) + len(stops['stations'])
print("> Amount of osm stops: " + str(amount_of_stops))
self.assertEqual(
amount_of_stops, self.required_variables['stops_osm_count'],
'Wrong count of stops in the cache file')
def test_refresh_stops_cache(self):
data = OsmConnector(self.config)
cache_file = os.path.join(self.data_dir, "accra-stops.pkl")
mocked_overpass_data_file = os.path.join(self.fixture_folder,
"overpass-stops.xml")
if os.path.isfile(cache_file):
os.remove(cache_file)
with patch("osm2gtfs.core.osm_connector.OsmConnector._query_stops"
) as mocked1:
overpass_xml = open(mocked_overpass_data_file, mode='r').read()
api = overpy.Overpass()
mocked1.return_value = api.parse_xml(overpass_xml)
data.get_stops(refresh=True)
self.assertTrue(os.path.isfile(cache_file),
'The stops cache file creation failed')
cache = Cache()
stops = cache.read_data('accra-stops')
amount_of_stops = len(stops['regular']) + len(stops['stations'])
self.assertEqual(amount_of_stops, 2529,
'Wrong count of stops in the cache file')
def test_refresh_routes_cache(self):
data = OsmConnector(self.config)
cache_file = os.path.join(self.standard_variables['data_dir'],
self.selector + "-routes.pkl")
mocked_overpass_data_file = self.standard_variables[
'mocked_overpass_routes']
if os.path.isfile(cache_file):
os.remove(cache_file)
with patch("osm2gtfs.core.osm_connector.OsmConnector._query_routes"
) as mocked1:
with open(mocked_overpass_data_file, mode='r') as ov:
overpass_xml = ov.read()
api = overpy.Overpass()
mocked1.return_value = api.parse_xml(overpass_xml)
data.get_routes(refresh=True)
self.assertTrue(os.path.isfile(cache_file),
'The routes cache file creation failed')
cache = Cache()
routes = cache.read_data(self.selector + "-routes")
# The Florianopolis creator eliminates (nine) routes in the Trips creator.
# This should be revised. Afterwards this overriden function can be removed.
self.assertEqual(len(routes),
self.required_variables['routes_count'] + 9,
'Wrong count of routes in the cache file')
def get_routes(self, refresh=False):
"""The get_routes function returns the data of routes from
OpenStreetMap converted into usable objects.
Data about routes is getting obtained from OpenStreetMap through the
Overpass API, based on the configuration from the config file.
Then this data gets prepared by building up objects of Line and
Itinerary objects that are related to each other.
It uses caching to leverage fast performance and spare the Overpass
API. Special commands are used to refresh cached data.
:param self: the own object including it's functions and variables
:param refresh: A simple boolean indicating a data refresh or use of
caching if possible.
:return routes: A dictionary of Line objects with related
Itinerary objects constituting the tree of data.
"""
# Preferably return cached data about routes
if refresh is False:
# Check if routes data is already built in this object
if not self.routes:
# If not, try to get routes data from file cache
self.routes = Cache.read_data(self.selector + '-routes')
# Return cached data if found
if bool(self.routes):
return self.routes
# No cached data was found or refresh was forced
logging.info("Query and build fresh data for routes")
# Obtain raw data about routes from OpenStreetMap
result = self._query_routes()
# Pre-sort relations by type
route_masters = {}
route_variants = {}
for relation in result.relations:
if relation.tags["type"] == "route_master":
route_masters[relation.id] = relation
else:
route_variants[relation.id] = relation
# Build routes from master relations
for rmid, route_master in route_masters.iteritems():
itineraries = OrderedDict()
# Build route variant members
for member in route_master.members:
# Create Itinerary objects from member route variants
if member.ref in route_variants:
rv = route_variants.pop(member.ref)
itinerary = self._build_itinerary(rv, result, route_master)
if itinerary is not None:
itineraries[rv.id] = itinerary
# Route variant was already used or is not valid
else:
rv = result.get_relations(member.ref)
if bool(rv):
rv = rv.pop()
logging.warning("Itinerary was assigned again:")
logging.warning("https://osm.org/relation/%s", rv.id)
itinerary = self._build_itinerary(
rv, result, route_master)
if itinerary is not None:
itineraries[rv.id] = itinerary
else:
logging.warning("Warning: This relation route master:")
logging.warning(" https://osm.org/relation/%s",
route_master.id)
logging.warning(
" has a member which is not a valid itinerary:")
logging.warning(" https://osm.org/relation/%s",
member.ref)
# Create Line object from route master
line = self._build_line(route_master, itineraries)
if line is None:
continue
# Make sure route_id (ref) number is not already taken
if line.route_id and line.route_id in [
elem.route_id for elem in self.routes.values()
]:
logging.warning("'Ref' of route_master already taken")
logging.warning(" https://osm.org/relation/%s",
route_master.id)
logging.warning(" Skipped. Please fix in OpenStreetMap")
continue
self.routes[str(line.osm_id)] = line
# Build routes from variants (missing master relation)
for rvid, route_variant in route_variants.iteritems():
logging.warning("Route (variant) without route_master")
logging.warning(" https://osm.org/relation/%s", route_variant.id)
logging.warning(" Please fix in OpenStreetMap")
itinerary = self._build_itinerary(route_variant, result, False)
# Make sure route_id (ref) number is not already taken
if itinerary is not None:
if itinerary.route_id in self.routes:
logging.warning("Route with existing route_id (ref)")
logging.warning(" https://osm.org/relation/%s",
route_variant.id)
logging.warning(" Skipped. Please fix in OpenStreetMap")
else:
# Create Line from route variant
itineraries = OrderedDict()
itineraries[itinerary.osm_id] = itinerary
line = self._build_line(route_variant, itineraries)
if line is not None:
self.routes[line.route_id] = line
# Cache data
Cache.write_data(self.selector + '-routes', self.routes)
return self.routes
def get_stops(self, refresh=False):
"""The get_stops function returns the data of stops and stop areas from
OpenStreetMap converted into usable objects.
Data about stops and stop_areas is getting obtained from OpenStreetMap
through the Overpass API, based on the configuration from the config
file.
Then this data gets prepared by building up objects of the class Stops
and Station (when the Stops are members of a stop_area)
It uses caching to leverage fast performance and spare the Overpass
API. Special commands are used to refresh cached data.
:param self: the own object including it's functions and variables
:param refresh: A simple boolean indicating a data refresh or use of
caching if possible.
:return stops: A dictionary of Stops and Stations constituting the
obtained data.
"""
# Preferably return cached data about stops
if refresh is False:
# Check if stops data is already built in this object
if not self.stops:
# If not, try to get stops data from file cache
self.stops = Cache.read_data(self.selector + '-stops')
if bool(self.stops):
# Maybe check for unnamed stop names
if self.auto_stop_names:
self._get_names_for_unnamed_stops()
# Return cached data if found
return self.stops
# No cached data was found or refresh was forced
logging.info("Query and build fresh data for stops")
# Obtain raw data about routes from OpenStreetMap
result = self._query_stops()
self.stops['regular'] = {}
self.stops['stations'] = {}
# Build stops from ways (polygons)
for stop in result.ways:
osm_type = "way"
stop_object = self._build_stop(stop, osm_type)
if stop_object:
self.stops['regular'][osm_type + "/" +
str(stop_object.osm_id)] = stop_object
# Build stops from nodes
for stop in result.nodes:
osm_type = "node"
stop_object = self._build_stop(stop, osm_type)
if stop_object:
self.stops['regular'][osm_type + "/" +
str(stop_object.osm_id)] = stop_object
# Build stations from stop_area relations
for stop in result.relations:
osm_type = "relation"
stop_object = self._build_station(stop, osm_type)
if stop_object:
self.stops['stations'][osm_type + "/" +
str(stop.id)] = stop_object
# Cache data
Cache.write_data(self.selector + '-stops', self.stops)
# Maybe check for unnamed stop names
if self.auto_stop_names:
self._get_names_for_unnamed_stops()
return self.stops
def get_schedule_source(self, refresh=False):
"""Loads the schedule source information.
Loads a schedule source file from either a path or a url specified
in the config file
:return schedule_source: The schedule read from a file.
"""
cached_file = self.data['selector'] + '-schedule'
if 'schedule_script' and 'schedule_csv' in self.data:
logging.info(
"Generating schedule using frequencies and script given in config file."
)
schedule_script = self.data['schedule_script']
schedule_csv = self.data['schedule_csv']
header_data = {
"start_date": self.data['feed_info']['start_date'],
"end_date": self.data['feed_info']['end_date'],
"included_lines": [],
"excluded_lines": []
}
script_dir = os.path.dirname(schedule_script)
script_name = os.path.splitext(
os.path.basename(schedule_script))[0]
sys.path.insert(0, script_dir)
gt = importlib.import_module(script_name)
schedule_json = gt.generate_json(gt.load_csv(schedule_csv),
header_data)
if 'schedule_source' in self.data:
logging.info("Writing schedule to {}.".format(
self.data['schedule_source']))
gt.write_json(schedule_json, self.data['schedule_source'])
schedule_source = json.dumps(schedule_json)
else:
schedule_source = json.dumps(schedule_json)
else:
if 'schedule_source' not in self.data:
return None
else:
source_file = self.data['schedule_source']
# Preferably return cached data about schedule
if refresh is False:
# Check if _schedule_source data is already present
if not self._schedule_source:
# If not, try to get _schedule_source from file cache
self._schedule_source = Cache.read_file(cached_file)
# Return cached data if found
if bool(self._schedule_source):
return self._schedule_source
# No cached data was found or refresh was forced
logging.info("Load schedule source information from %s",
source_file)
# Check if local file exists
if os.path.isfile(source_file):
# Open file and add to config object
with open(source_file, 'r') as f:
schedule_source = f.read()
else:
# Check if it is a valid url
try:
schedule_source_file = urlopen(source_file)
except ValueError:
logging.error("Couldn't find schedule_source file.")
sys.exit(0)
schedule_source = schedule_source_file.read()
self._schedule_source = schedule_source
# Cache data
Cache.write_file(cached_file, self._schedule_source)
return self._schedule_source